feat: reland vault w/o pgsodium

ansible/files/admin_api_scripts/pg_upgrade_scripts/complete.sh

@@ -150,6 +150,43 @@ EOF
 
         run_sql -c "$PATCH_PGMQ_QUERY"
         run_sql -c "update pg_extension set extowner = 'postgres'::regrole where extname = 'pgmq';"
+
+        # Patch to handle upgrading to pgsodium-less Vault
+        REENCRYPT_VAULT_SECRETS_QUERY=$(cat <<EOF
+        DO \$\$
+        BEGIN
+          IF EXISTS (SELECT FROM pg_available_extension_versions WHERE name = 'supabase_vault' AND version = '0.3.0')
+            AND EXISTS (SELECT FROM pg_extension WHERE extname = 'supabase_vault')
+          THEN
+            IF (SELECT extversion FROM pg_extension WHERE extname = 'supabase_vault') != '0.2.8' THEN
+              GRANT USAGE ON SCHEMA vault TO postgres WITH GRANT OPTION;
+              GRANT SELECT, DELETE ON vault.secrets, vault.decrypted_secrets TO postgres WITH GRANT OPTION;
+              GRANT EXECUTE ON FUNCTION vault.create_secret, vault.update_secret, vault._crypto_aead_det_decrypt TO postgres WITH GRANT OPTION;
+            END IF;
+            -- Do an explicit IF EXISTS check to avoid referencing pgsodium objects if the project already migrated away from using pgsodium.
+            IF EXISTS (SELECT FROM vault.secrets WHERE key_id IS NOT NULL) THEN
+              UPDATE vault.secrets s
+              SET
+                secret = encode(
+                  vault._crypto_aead_det_encrypt(
+                    message := pgsodium.crypto_aead_det_decrypt(decode(s.secret, 'base64'), convert_to(s.id || s.description || s.created_at || s.updated_at, 'utf8'), s.key_id, s.nonce),
+                    additional := convert_to(s.id::text, 'utf8'),
+                    key_id := 0,
+                    context := 'pgsodium'::bytea,
+                    nonce := s.nonce
+                  ),
+                  'base64'
+                ),
+                key_id = NULL
+              WHERE
+                key_id IS NOT NULL;
+            END IF;
+          END IF;
+        END
+        \$\$;
+EOF
+        )
+        run_sql -c "$REENCRYPT_VAULT_SECRETS_QUERY"
     fi
 
     run_sql -c "grant pg_read_all_data, pg_signal_backend to postgres"

ansible/files/postgresql_config/postgresql.conf.j2

@@ -688,7 +688,7 @@ default_text_search_config = 'pg_catalog.english'
 #local_preload_libraries = ''
 #session_preload_libraries = ''
 
-shared_preload_libraries = 'pg_stat_statements, pgaudit, plpgsql, plpgsql_check, pg_cron, pg_net, pgsodium, timescaledb, auto_explain, pg_tle, plan_filter'	# (change requires restart)
+shared_preload_libraries = 'pg_stat_statements, pgaudit, plpgsql, plpgsql_check, pg_cron, pg_net, pgsodium, timescaledb, auto_explain, pg_tle, plan_filter, supabase_vault'	# (change requires restart)
 jit_provider = 'llvmjit'		# JIT library to use
 
 # - Other Defaults -

ansible/files/postgresql_extension_custom_scripts/supabase_vault/after-create.sql

@@ -0,0 +1,3 @@
+grant usage on schema vault to postgres with grant option;
+grant select, delete on vault.secrets, vault.decrypted_secrets to postgres with grant option;
+grant execute on function vault.create_secret, vault.update_secret, vault._crypto_aead_det_decrypt to postgres with grant option;

ansible/tasks/test-image.yml

@@ -11,17 +11,34 @@
 #     cmd: sed -i.bak -e "s/pg_net,\ pgsodium,\ timescaledb/pg_net,\ timescaledb/g" -e "s/pgsodium.getkey_script=/#pgsodium.getkey_script=/g" /etc/postgresql/postgresql.conf
 #   when: debpkg_mode or stage2_nix
 
-- name: Temporarily disable PG Sodium references in config
+- name: Temporarily disable PG Sodium and Supabase Vault references in config
   become: yes
   become_user: postgres
   shell:
     cmd: >
-      sed -i.bak
-      -e 's/\(shared_preload_libraries = '\''.*\)pgsodium,\(.*'\''\)/\1\2/'
+      sed -i.bak 
+      -e 's/\(shared_preload_libraries = '\''.*\)pgsodium,\(.*'\''\)/\1\2/' 
+      -e 's/\(shared_preload_libraries = '\''.*\)supabase_vault,\(.*'\''\)/\1\2/' 
+      -e 's/\(shared_preload_libraries = '\''.*\), *supabase_vault'\''/\1'\''/'
       -e 's/pgsodium.getkey_script=/#pgsodium.getkey_script=/'
       /etc/postgresql/postgresql.conf
   when: debpkg_mode or stage2_nix
 
+- name: Verify pgsodium and vault removal from config
+  become: yes
+  become_user: postgres
+  shell:
+    cmd: |
+      FOUND=$(grep -E "shared_preload_libraries.*pgsodium|shared_preload_libraries.*supabase_vault|^pgsodium\.getkey_script" /etc/postgresql/postgresql.conf)
+      if [ ! -z "$FOUND" ]; then
+        echo "Found unremoved references:"
+        echo "$FOUND"
+        exit 1
+      fi
+  register: verify_result
+  failed_when: verify_result.rc != 0
+  when: debpkg_mode or stage2_nix
+
 - name: Start Postgres Database to load all extensions.
   become: yes
   become_user: postgres

ansible/vars.yml

@@ -8,8 +8,8 @@ postgres_major:
 
 # Full version strings for each major version
 postgres_release:
-  postgresorioledb-17: "17.0.1.036-orioledb"
-  postgres15: "15.8.1.040"
+  postgresorioledb-17: "17.0.1.036-orioledb-vault-1"
+  postgres15: "15.8.1.040-vault-1"
 
 # Non Postgres Extensions
 pgbouncer_release: "1.19.0"

ebssurrogate/files/unit-tests/unit-test-01.sql

@@ -17,7 +17,6 @@ BEGIN
         extension_array := ARRAY[
             'plpgsql',
             'pg_stat_statements',
-            'pgsodium',
             'pgtap',
             'pg_graphql',
             'pgcrypto',
@@ -30,7 +29,6 @@ BEGIN
         extension_array := ARRAY[
             'plpgsql',
             'pg_stat_statements',
-            'pgsodium',
             'pgtap',
             'pg_graphql',
             'pgcrypto',
@@ -44,7 +42,7 @@ BEGIN
     PERFORM set_config('myapp.extensions', array_to_string(extension_array, ','), false);
 END $$;
 
-SELECT plan(8);
+SELECT no_plan();
 
 SELECT extensions_are(
     string_to_array(current_setting('myapp.extensions'), ',')::text[]
@@ -56,9 +54,5 @@ SELECT has_schema('pg_catalog');
 SELECT has_schema('information_schema');
 SELECT has_schema('public');
 
-SELECT function_privs_are('pgsodium', 'crypto_aead_det_decrypt', array['bytea', 'bytea', 'uuid', 'bytea'], 'service_role', array['EXECUTE']);
-SELECT function_privs_are('pgsodium', 'crypto_aead_det_encrypt', array['bytea', 'bytea', 'uuid', 'bytea'], 'service_role', array['EXECUTE']);
-SELECT function_privs_are('pgsodium', 'crypto_aead_det_keygen', array[]::text[], 'service_role', array['EXECUTE']);
-
 SELECT * FROM finish();
-ROLLBACK;
+ROLLBACK;

flake.nix

@@ -571,42 +571,49 @@
             sqlTests = ./nix/tests/smoke;
             pg_prove = pkgs.perlPackages.TAPParserSourceHandlerpgTAP;
             pg_regress = basePackages.pg_regress;
-            getkey-script = pkgs.writeScriptBin "pgsodium-getkey" ''
-              #!${pkgs.bash}/bin/bash
-              set -euo pipefail
-
-              TMPDIR_BASE=$(mktemp -d)
-
-              if [[ "$(uname)" == "Darwin" ]]; then
-                KEY_DIR="/private/tmp/pgsodium"
-              else
-                KEY_DIR="''${PGSODIUM_KEY_DIR:-$TMPDIR_BASE/pgsodium}"
-              fi
-              KEY_FILE="$KEY_DIR/pgsodium.key"
-
-              if ! mkdir -p "$KEY_DIR" 2>/dev/null; then
-                echo "Error: Could not create key directory $KEY_DIR" >&2
-                exit 1
-              fi
-              chmod 1777 "$KEY_DIR"
-
-              if [[ ! -f "$KEY_FILE" ]]; then
-                if ! (dd if=/dev/urandom bs=32 count=1 2>/dev/null | od -A n -t x1 | tr -d ' \n' > "$KEY_FILE"); then
-                  if ! (openssl rand -hex 32 > "$KEY_FILE"); then
-                    echo "00000000000000000000000000000000" > "$KEY_FILE"
-                    echo "Warning: Using fallback key" >&2
+            getkey-script = pkgs.stdenv.mkDerivation {
+              name = "pgsodium-getkey";
+              buildCommand = ''
+                mkdir -p $out/bin
+                cat > $out/bin/pgsodium-getkey << 'EOF'
+                #!${pkgs.bash}/bin/bash
+                set -euo pipefail
+
+                TMPDIR_BASE=$(mktemp -d)
+
+                if [[ "$(uname)" == "Darwin" ]]; then
+                  KEY_DIR="/private/tmp/pgsodium"
+                else
+                  KEY_DIR="''${PGSODIUM_KEY_DIR:-$TMPDIR_BASE/pgsodium}"
+                fi
+                KEY_FILE="$KEY_DIR/pgsodium.key"
+
+                if ! mkdir -p "$KEY_DIR" 2>/dev/null; then
+                  echo "Error: Could not create key directory $KEY_DIR" >&2
+                  exit 1
+                fi
+                chmod 1777 "$KEY_DIR"
+
+                if [[ ! -f "$KEY_FILE" ]]; then
+                  if ! (dd if=/dev/urandom bs=32 count=1 2>/dev/null | od -A n -t x1 | tr -d ' \n' > "$KEY_FILE"); then
+                    if ! (openssl rand -hex 32 > "$KEY_FILE"); then
+                      echo "00000000000000000000000000000000" > "$KEY_FILE"
+                      echo "Warning: Using fallback key" >&2
+                    fi
                   fi
+                  chmod 644 "$KEY_FILE"
                 fi
-                chmod 644 "$KEY_FILE"
-              fi
-
-              if [[ -f "$KEY_FILE" && -r "$KEY_FILE" ]]; then
-                cat "$KEY_FILE"
-              else
-                echo "Error: Cannot read key file $KEY_FILE" >&2
-                exit 1
-              fi
-            '';
+
+                if [[ -f "$KEY_FILE" && -r "$KEY_FILE" ]]; then
+                  cat "$KEY_FILE"
+                else
+                  echo "Error: Cannot read key file $KEY_FILE" >&2
+                  exit 1
+                fi
+                EOF
+                chmod +x $out/bin/pgsodium-getkey
+              '';
+            };
 
             # Use the shared setup but with a test-specific name
             start-postgres-server-bin = makePostgresDevSetup {
@@ -675,6 +682,8 @@
               echo "listen_addresses = '*'" >> "$PGTAP_CLUSTER"/postgresql.conf
               echo "port = 5435" >> "$PGTAP_CLUSTER"/postgresql.conf
               echo "host all all 127.0.0.1/32 trust" >> $PGTAP_CLUSTER/pg_hba.conf
+              echo "Checking shared_preload_libraries setting:"
+              grep -rn "shared_preload_libraries" "$PGTAP_CLUSTER"/postgresql.conf
               # Remove timescaledb if running orioledb-17 check
               echo "I AM ${pgpkg.version}===================================================="
               if [[ "${pgpkg.version}" == *"17"* ]]; then

migrations/db/migrations/20250218031949_pgsodium_mask_role.sql

@@ -1,25 +1,31 @@
 -- migrate:up
-CREATE OR REPLACE FUNCTION pgsodium.mask_role(masked_role regrole, source_name text, view_name text)
-RETURNS void
-LANGUAGE plpgsql
-SECURITY DEFINER
-SET search_path TO ''
-AS $function$
+
+DO $$
 BEGIN
-  EXECUTE format(
-    'GRANT SELECT ON pgsodium.key TO %s',
-    masked_role);
+  IF EXISTS (SELECT FROM pg_extension WHERE extname = 'pgsodium') THEN
+    CREATE OR REPLACE FUNCTION pgsodium.mask_role(masked_role regrole, source_name text, view_name text)
+    RETURNS void
+    LANGUAGE plpgsql
+    SECURITY DEFINER
+    SET search_path TO ''
+    AS $function$
+    BEGIN
+      EXECUTE format(
+        'GRANT SELECT ON pgsodium.key TO %s',
+        masked_role);
 
-  EXECUTE format(
-    'GRANT pgsodium_keyiduser, pgsodium_keyholder TO %s',
-    masked_role);
+      EXECUTE format(
+        'GRANT pgsodium_keyiduser, pgsodium_keyholder TO %s',
+        masked_role);
 
-  EXECUTE format(
-    'GRANT ALL ON %I TO %s',
-    view_name,
-    masked_role);
-  RETURN;
-END
-$function$;
+      EXECUTE format(
+        'GRANT ALL ON %I TO %s',
+        view_name,
+        masked_role);
+      RETURN;
+    END
+    $function$;
+  END IF;
+END $$;
 
 -- migrate:down

nix/ext/vault.nix

@@ -1,23 +1,24 @@
-{ lib, stdenv, fetchFromGitHub, postgresql }:
+{ lib, stdenv, fetchFromGitHub, libsodium, postgresql }:
 
 stdenv.mkDerivation rec {
   pname = "vault";
-  version = "0.2.9";
+  version = "0.3.1";
 
-  buildInputs = [ postgresql ];
+  buildInputs = [ libsodium postgresql ];
 
   src = fetchFromGitHub {
     owner = "supabase";
     repo = pname;
     rev = "refs/tags/v${version}";
-    hash = "sha256-kXTngBW4K6FkZM8HvJG2Jha6OQqbejhnk7tchxy031I=";
+    hash = "sha256-MC87bqgtynnDhmNZAu96jvfCpsGDCPB0g5TZfRQHd30=";
   };
 
   installPhase = ''
     mkdir -p $out/{lib,share/postgresql/extension}
 
-    cp sql/*.sql $out/share/postgresql/extension
-    cp *.control $out/share/postgresql/extension
+    install -D *${postgresql.dlSuffix} $out/lib
+    install -D -t $out/share/postgresql/extension sql/*.sql
+    install -D -t $out/share/postgresql/extension *.control
   '';
 
   meta = with lib; {