Skip to content

Commit dfa63f8

Browse files
committed
Add support for seamless DB upgrades / migrations (--upgrade).
1 parent b4eb5ad commit dfa63f8

File tree

8 files changed

+214
-4
lines changed

8 files changed

+214
-4
lines changed

.gitignore

-1
Original file line numberDiff line numberDiff line change
@@ -1 +0,0 @@
1-
dictpress

cmd/dictpress/install.go

+17-1
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,11 @@ import (
44
"fmt"
55
"os"
66
"strings"
7+
8+
"github.com/jmoiron/sqlx"
79
)
810

9-
func installSchema(app *App, prompt bool) {
11+
func installSchema(ver string, app *App, prompt bool) {
1012
if prompt {
1113
fmt.Println("")
1214
fmt.Println("** first time installation **")
@@ -39,5 +41,19 @@ func installSchema(app *App, prompt bool) {
3941
return
4042
}
4143

44+
// Insert the current migration version.
45+
if err := recordMigrationVersion(ver, app.db); err != nil {
46+
app.lo.Fatal(err)
47+
}
48+
4249
app.lo.Println("successfully installed schema")
4350
}
51+
52+
// recordMigrationVersion inserts the given version (of DB migration) into the
53+
// `migrations` array in the settings table.
54+
func recordMigrationVersion(ver string, db *sqlx.DB) error {
55+
_, err := db.Exec(fmt.Sprintf(`INSERT INTO settings (key, value)
56+
VALUES('migrations', '["%s"]'::JSONB)
57+
ON CONFLICT (key) DO UPDATE SET value = settings.value || EXCLUDED.value`, ver))
58+
return err
59+
}

cmd/dictpress/main.go

+11-2
Original file line numberDiff line numberDiff line change
@@ -84,8 +84,9 @@ func init() {
8484
"path to one or more config files (will be merged in order)")
8585
f.String("site", "", "path to a site theme. If left empty, only HTTP APIs will be available.")
8686
f.Bool("install", false, "run first time DB installation")
87+
f.Bool("upgrade", false, "upgrade database to the current version")
88+
f.Bool("yes", false, "assume 'yes' to prompts during --install/upgrade")
8789
f.String("import", "", "import a CSV file into the database. eg: --import=data.csv")
88-
f.Bool("yes", false, "assume 'yes' to prompts, eg: during --install")
8990
f.Bool("version", false, "current version of the build")
9091

9192
if err := f.Parse(os.Args[1:]); err != nil {
@@ -143,10 +144,18 @@ func main() {
143144

144145
// Install schema.
145146
if ko.Bool("install") {
146-
installSchema(app, !ko.Bool("yes"))
147+
installSchema(migList[len(migList)-1].version, app, !ko.Bool("yes"))
147148
return
148149
}
149150

151+
if ko.Bool("upgrade") {
152+
upgrade(db, app.fs, !ko.Bool("yes"))
153+
os.Exit(0)
154+
}
155+
156+
// Before the queries are prepared, see if there are pending upgrades.
157+
checkUpgrade(db)
158+
150159
// Load SQL queries.
151160
qB, err := app.fs.Read("/queries.sql")
152161
if err != nil {

cmd/dictpress/upgrade.go

+147
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
package main
2+
3+
import (
4+
"fmt"
5+
"strings"
6+
7+
"github.com/jmoiron/sqlx"
8+
"github.com/knadh/dictpress/internal/migrations"
9+
"github.com/knadh/koanf/v2"
10+
"github.com/knadh/stuffbin"
11+
"github.com/lib/pq"
12+
"golang.org/x/mod/semver"
13+
)
14+
15+
// migFunc represents a migration function for a particular version.
16+
// fn (generally) executes database migrations and additionally
17+
// takes the filesystem and config objects in case there are additional bits
18+
// of logic to be performed before executing upgrades. fn is idempotent.
19+
type migFunc struct {
20+
version string
21+
fn func(*sqlx.DB, stuffbin.FileSystem, *koanf.Koanf) error
22+
}
23+
24+
// migList is the list of available migList ordered by the semver.
25+
// Each migration is a Go file in internal/migrations named after the semver.
26+
// The functions are named as: v0.7.0 => migrations.V0_7_0() and are idempotent.
27+
var migList = []migFunc{
28+
{"v2.0.0", migrations.V2_0_0},
29+
}
30+
31+
// upgrade upgrades the database to the current version by running SQL migration files
32+
// for all version from the last known version to the current one.
33+
func upgrade(db *sqlx.DB, fs stuffbin.FileSystem, prompt bool) {
34+
if prompt {
35+
var ok string
36+
fmt.Printf("** IMPORTANT: Take a backup of the database before upgrading.\n")
37+
fmt.Print("continue (y/n)? ")
38+
if _, err := fmt.Scanf("%s", &ok); err != nil {
39+
lo.Fatalf("error reading value from terminal: %v", err)
40+
}
41+
if strings.ToLower(ok) != "y" {
42+
fmt.Println("upgrade cancelled")
43+
return
44+
}
45+
}
46+
47+
_, toRun, err := getPendingMigrations(db)
48+
if err != nil {
49+
lo.Fatalf("error checking migrations: %v", err)
50+
}
51+
52+
// No migrations to run.
53+
if len(toRun) == 0 {
54+
lo.Printf("no upgrades to run. Database is up to date.")
55+
return
56+
}
57+
58+
// Execute migrations in succession.
59+
for _, m := range toRun {
60+
lo.Printf("running migration %s", m.version)
61+
if err := m.fn(db, fs, ko); err != nil {
62+
lo.Fatalf("error running migration %s: %v", m.version, err)
63+
}
64+
65+
// Record the migration version in the settings table. There was no
66+
// settings table until v0.7.0, so ignore the no-table errors.
67+
if err := recordMigrationVersion(m.version, db); err != nil {
68+
if isTableNotExistErr(err) {
69+
continue
70+
}
71+
lo.Fatalf("error recording migration version %s: %v", m.version, err)
72+
}
73+
}
74+
75+
lo.Printf("upgrade complete")
76+
}
77+
78+
// checkUpgrade checks if the current database schema matches the expected
79+
// binary version.
80+
func checkUpgrade(db *sqlx.DB) {
81+
lastVer, toRun, err := getPendingMigrations(db)
82+
if err != nil {
83+
lo.Fatalf("error checking migrations: %v", err)
84+
}
85+
86+
// No migrations to run.
87+
if len(toRun) == 0 {
88+
return
89+
}
90+
91+
var vers []string
92+
for _, m := range toRun {
93+
vers = append(vers, m.version)
94+
}
95+
96+
lo.Fatalf(`there are %d pending database upgrade(s): %v. The last upgrade was %s. Backup the database and run --upgrade`,
97+
len(toRun), vers, lastVer)
98+
}
99+
100+
// getPendingMigrations gets the pending migrations by comparing the last
101+
// recorded migration in the DB against all migrations listed in `migrations`.
102+
func getPendingMigrations(db *sqlx.DB) (string, []migFunc, error) {
103+
lastVer, err := getLastMigrationVersion(db)
104+
if err != nil {
105+
return "", nil, err
106+
}
107+
108+
// Iterate through the migration versions and get everything above the last
109+
// upgraded semver.
110+
var toRun []migFunc
111+
for i, m := range migList {
112+
if semver.Compare(m.version, lastVer) > 0 {
113+
toRun = migList[i:]
114+
break
115+
}
116+
}
117+
118+
return lastVer, toRun, nil
119+
}
120+
121+
// getLastMigrationVersion returns the last migration semver recorded in the DB.
122+
// If there isn't any, `v0.0.0` is returned.
123+
func getLastMigrationVersion(db *sqlx.DB) (string, error) {
124+
var v string
125+
if err := db.Get(&v, `
126+
SELECT COALESCE(
127+
(SELECT value->>-1 FROM settings WHERE key='migrations'),
128+
'v0.0.0')`); err != nil {
129+
if isTableNotExistErr(err) {
130+
return "v0.0.0", nil
131+
}
132+
return v, err
133+
}
134+
return v, nil
135+
}
136+
137+
// isTableNotExistErr checks if the given error represents a Postgres/pq
138+
// "table does not exist" error.
139+
func isTableNotExistErr(err error) bool {
140+
if p, ok := err.(*pq.Error); ok {
141+
// `settings` table does not exist. It was introduced in v0.7.0.
142+
if p.Code == "42P01" {
143+
return true
144+
}
145+
}
146+
return false
147+
}

go.mod

+1
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ require (
1818
github.com/lib/pq v1.10.9
1919
github.com/spf13/pflag v1.0.5
2020
gitlab.com/joice/mlphone-go v0.0.0-20201001084309-2bb02984eed8
21+
golang.org/x/mod v0.8.0
2122
gopkg.in/volatiletech/null.v6 v6.0.0-20170828023728-0bef4e07ae1b
2223
)
2324

go.sum

+2
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,8 @@ golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4
111111
golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
112112
golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
113113
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
114+
golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8=
115+
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
114116
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
115117
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
116118
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=

internal/migrations/v2_0_0.go

+27
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
package migrations
2+
3+
import (
4+
"github.com/jmoiron/sqlx"
5+
"github.com/knadh/koanf/v2"
6+
"github.com/knadh/stuffbin"
7+
)
8+
9+
// V2_0_0 performs the DB migrations.
10+
func V2_0_0(db *sqlx.DB, fs stuffbin.FileSystem, ko *koanf.Koanf) error {
11+
if _, err := db.Exec(`
12+
CREATE TABLE IF NOT EXISTS settings (
13+
key TEXT NOT NULL UNIQUE,
14+
value JSONB NOT NULL DEFAULT '{}',
15+
updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
16+
);
17+
CREATE INDEX IF NOT EXISTS idx_settings_key ON settings(key);
18+
`); err != nil {
19+
return err
20+
}
21+
22+
if _, err := db.Exec(`ALTER TABLE entries ADD COLUMN IF NOT EXISTS meta JSONB NOT NULL DEFAULT '{}'`); err != nil {
23+
return err
24+
}
25+
26+
return nil
27+
}

schema.sql

+9
Original file line numberDiff line numberDiff line change
@@ -78,3 +78,12 @@ CREATE TABLE comments (
7878

7979
created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
8080
);
81+
82+
-- settings
83+
DROP TABLE IF EXISTS settings CASCADE;
84+
CREATE TABLE settings (
85+
key TEXT NOT NULL UNIQUE,
86+
value JSONB NOT NULL DEFAULT '{}',
87+
updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
88+
);
89+
DROP INDEX IF EXISTS idx_settings_key; CREATE INDEX idx_settings_key ON settings(key);

0 commit comments

Comments
 (0)