Skip to content

Commit e89d526

Browse files
TylerHilleryTylerHillery
authored
fix: make storage migrations idempotent (#805)
* fix: make storage migrations idempotent * fix: add PGRST_JWT_SECRET to github action * fix: add drop function if exists cascade to search_v2 * fix: iceberg catalog id fkey creation on if column already exists * fix: drop functions * feat: add ci check that pg dump is the same before and after running migrations * refactor: add restric key to pg dump * fix: minor formatting changes * refactor: change from drop table to truncate * test: add migration idempotency validation
1 parent cce047b commit e89d526

14 files changed

+131
-25
lines changed

‎.github/workflows/ci.yml‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,30 @@ jobs:
9595
with:
9696
github-token: ${{ secrets.GITHUB_TOKEN }}
9797

98+
- name: Verify migration idempotency
99+
run: |
100+
pg_dump "$DATABASE_URL" \
101+
--exclude-table-data=storage.migrations \
102+
--restrict-key=test \
103+
> before.sql
104+
105+
npm run migration:test-idempotency
106+
107+
pg_dump "$DATABASE_URL" \
108+
--exclude-table-data=storage.migrations \
109+
--restrict-key=test \
110+
> after.sql
111+
112+
diff before.sql after.sql || (echo 'Schema mismatch!'; exit 1)
113+
114+
env:
115+
PGRST_JWT_SECRET: ${{ secrets.PGRST_JWT_SECRET }}
116+
DATABASE_URL: postgresql://postgres:[email protected]/postgres
117+
DB_INSTALL_ROLES: true
118+
ENABLE_DEFAULT_METRICS: false
119+
PG_QUEUE_ENABLE: false
120+
MULTI_TENANT: false
121+
98122
- name: Ensure OrioleDB migration compatibility
99123
run: |
100124
npm run infra:restart:oriole

‎migrations/tenant/00010-search-files-search-function.sql‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
drop function storage.search;
21

32
create or replace function storage.search (
43
prefix text,

‎migrations/tenant/0002-storage-schema.sql‎

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,23 @@ BEGIN
1818
END IF;
1919

2020
-- Install ROLES
21-
EXECUTE 'CREATE ROLE ' || anon_role || ' NOLOGIN NOINHERIT';
22-
EXECUTE 'CREATE ROLE ' || authenticated_role || ' NOLOGIN NOINHERIT';
23-
EXECUTE 'CREATE ROLE ' || service_role || ' NOLOGIN NOINHERIT bypassrls';
21+
IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = anon_role) THEN
22+
EXECUTE 'CREATE ROLE ' || anon_role || ' NOLOGIN NOINHERIT';
23+
END IF;
24+
25+
IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = authenticated_role ) THEN
26+
EXECUTE 'CREATE ROLE ' || authenticated_role || ' NOLOGIN NOINHERIT';
27+
END IF;
28+
29+
IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = service_role) THEN
30+
EXECUTE 'CREATE ROLE ' || service_role || ' NOLOGIN NOINHERIT bypassrls';
31+
END IF;
32+
33+
IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'authenticator') THEN
34+
EXECUTE 'CREATE USER authenticator NOINHERIT';
35+
END IF;
36+
2437

25-
create user authenticator noinherit;
2638
EXECUTE 'grant ' || anon_role || ' to authenticator';
2739
EXECUTE 'grant ' || authenticated_role || ' to authenticator';
2840
EXECUTE 'grant ' || service_role || ' to authenticator';
@@ -70,7 +82,6 @@ CREATE INDEX IF NOT EXISTS name_prefix_search ON storage.objects(name text_patte
7082

7183
ALTER TABLE storage.objects ENABLE ROW LEVEL SECURITY;
7284

73-
drop function if exists storage.foldername;
7485
CREATE OR REPLACE FUNCTION storage.foldername(name text)
7586
RETURNS text[]
7687
LANGUAGE plpgsql
@@ -83,7 +94,6 @@ BEGIN
8394
END
8495
$function$;
8596

86-
drop function if exists storage.filename;
8797
CREATE OR REPLACE FUNCTION storage.filename(name text)
8898
RETURNS text
8999
LANGUAGE plpgsql
@@ -96,7 +106,6 @@ BEGIN
96106
END
97107
$function$;
98108

99-
drop function if exists storage.extension;
100109
CREATE OR REPLACE FUNCTION storage.extension(name text)
101110
RETURNS text
102111
LANGUAGE plpgsql
@@ -113,7 +122,6 @@ END
113122
$function$;
114123

115124
-- @todo can this query be optimised further?
116-
drop function if exists storage.search;
117125
CREATE OR REPLACE FUNCTION storage.search(prefix text, bucketname text, limits int DEFAULT 100, levels int DEFAULT 1, offsets int DEFAULT 0)
118126
RETURNS TABLE (
119127
name text,

‎migrations/tenant/0006-change-column-name-in-get-size.sql‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
DROP FUNCTION storage.get_size_by_bucket();
1+
DROP FUNCTION IF EXISTS storage.get_size_by_bucket();
22
CREATE OR REPLACE FUNCTION storage.get_size_by_bucket()
33
RETURNS TABLE (
44
size BIGINT,

‎migrations/tenant/0009-fix-search-function.sql‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
drop function if exists storage.search;
21
CREATE OR REPLACE FUNCTION storage.search(prefix text, bucketname text, limits int DEFAULT 100, levels int DEFAULT 1, offsets int DEFAULT 0)
32
RETURNS TABLE (
43
name text,

‎migrations/tenant/0038-iceberg-catalog-flag-on-buckets.sql‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,9 @@ DO $$
4040
updated_at timestamptz NOT NULL default now()
4141
);
4242

43-
CREATE UNIQUE INDEX IF NOT EXISTS idx_iceberg_namespaces_bucket_id ON storage.iceberg_namespaces (bucket_id, name);
43+
IF EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'iceberg_namespaces' AND column_name = 'bucket_id') THEN
44+
CREATE UNIQUE INDEX IF NOT EXISTS idx_iceberg_namespaces_bucket_id ON storage.iceberg_namespaces (bucket_id, name);
45+
END IF;
4446

4547
CREATE TABLE IF NOT EXISTS storage.iceberg_tables (
4648
id uuid primary key default gen_random_uuid(),
@@ -52,6 +54,7 @@ DO $$
5254
updated_at timestamptz NOT NULL default now()
5355
);
5456

57+
DROP INDEX IF EXISTS idx_iceberg_tables_namespace_id;
5558
CREATE UNIQUE INDEX idx_iceberg_tables_namespace_id ON storage.iceberg_tables (namespace_id, name);
5659

5760
ALTER TABLE storage.iceberg_namespaces ENABLE ROW LEVEL SECURITY;

‎migrations/tenant/0039-add-search-v2-sort-support.sql‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
DROP FUNCTION IF EXISTS storage.search_v2;
21
CREATE OR REPLACE FUNCTION storage.search_v2 (
32
prefix text,
43
bucket_name text,

‎migrations/tenant/0040-fix-prefix-race-conditions-optimized.sql‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -240,6 +240,10 @@ BEGIN
240240
END;
241241
$$;
242242

243+
DROP TRIGGER IF EXISTS objects_delete_cleanup ON storage.objects;
244+
DROP TRIGGER IF EXISTS prefixes_delete_cleanup ON storage.prefixes;
245+
DROP TRIGGER IF EXISTS objects_update_cleanup ON storage.objects;
246+
243247
-- Trigger bindings
244248
CREATE TRIGGER objects_delete_cleanup
245249
AFTER DELETE ON storage.objects

‎migrations/tenant/0048-iceberg-catalog-ids.sql‎

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -54,8 +54,28 @@ DO $$
5454
ALTER TABLE storage.iceberg_tables RENAME COLUMN bucket_id to bucket_name;
5555
END IF;
5656

57-
ALTER TABLE storage.iceberg_namespaces ADD COLUMN IF NOT EXISTS catalog_id uuid NULL REFERENCES storage.buckets_analytics(id) ON DELETE CASCADE;
58-
ALTER TABLE storage.iceberg_tables ADD COLUMN IF NOT EXISTS catalog_id uuid NULL REFERENCES storage.buckets_analytics(id) ON DELETE CASCADE;
57+
ALTER TABLE storage.iceberg_namespaces ADD COLUMN IF NOT EXISTS catalog_id uuid NULL;
58+
ALTER TABLE storage.iceberg_tables ADD COLUMN IF NOT EXISTS catalog_id uuid NULL;
59+
60+
IF NOT EXISTS (
61+
SELECT 1 FROM information_schema.table_constraints
62+
WHERE table_schema = 'storage'
63+
AND table_name = 'iceberg_namespaces'
64+
AND constraint_name = 'iceberg_namespaces_catalog_id_fkey'
65+
) THEN
66+
ALTER TABLE storage.iceberg_namespaces ADD CONSTRAINT iceberg_namespaces_catalog_id_fkey
67+
FOREIGN KEY (catalog_id) REFERENCES storage.buckets_analytics(id) ON DELETE CASCADE;
68+
END IF;
69+
70+
IF NOT EXISTS (
71+
SELECT 1 FROM information_schema.table_constraints
72+
WHERE table_schema = 'storage'
73+
AND table_name = 'iceberg_tables'
74+
AND constraint_name = 'iceberg_tables_catalog_id_fkey'
75+
) THEN
76+
ALTER TABLE storage.iceberg_tables ADD CONSTRAINT iceberg_tables_catalog_id_fkey
77+
FOREIGN KEY (catalog_id) REFERENCES storage.buckets_analytics(id) ON DELETE CASCADE;
78+
END IF;
5979

6080
CREATE UNIQUE INDEX IF NOT EXISTS idx_iceberg_namespaces_bucket_id ON storage.iceberg_namespaces (catalog_id, name);
6181
CREATE UNIQUE INDEX IF NOT EXISTS idx_iceberg_tables_namespace_id ON storage.iceberg_tables (catalog_id, namespace_id, name);

‎package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
"format": "prettier -c --write src/**",
1111
"lint": "prettier -v && prettier -c src/**",
1212
"migration:run": "tsx src/scripts/migrate-call.ts",
13+
"migration:test-idempotency": "tsx src/scripts/test-migration-idempotency.ts",
1314
"migrations:types": "tsx src/scripts/migrations-types.ts",
1415
"docs:export": "tsx ./src/scripts/export-docs.ts",
1516
"test:dummy-data": "tsx -r dotenv/config ./src/test/db/import-dummy-data.ts",

0 commit comments

Comments
 (0)