Merge branch 'develop' into cs/feat-config-reloads

.github/workflows/publish-nix-pgupgrade-scripts.yml

@@ -72,6 +72,9 @@ jobs:
         id: process_release_version
         run: |
           VERSION=$(grep 'postgres-version' common-nix.vars.pkr.hcl | sed -e 's/postgres-version = "\(.*\)"/\1/g')
+          if [[ "${{ inputs.postgresVersion }}" != "" ]]; then
+            VERSION=${{ inputs.postgresVersion }}
+          fi
           echo "version=$VERSION" >> "$GITHUB_OUTPUT"
 
       - name: Create a tarball containing pg_upgrade scripts

.github/workflows/testinfra-nix.yml

@@ -65,12 +65,12 @@ jobs:
       - name: Cleanup resources on build cancellation
         if: ${{ cancelled() }}
         run: |
-          aws ec2 --region ap-southeast-1 describe-instances --filters "Name=tag:packerExecutionId,Values=${GITHUB_RUN_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -n 1 -I {} aws ec2 terminate-instances --region ap-southeast-1 --instance-ids {}
-   
-      - name: Cleanup resources on build cancellation
+          aws ec2 --region ap-southeast-1 describe-instances --filters "Name=tag:packerExecutionId,Values=${GITHUB_RUN_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --region ap-southeast-1 --instance-ids
+
+      - name: Cleanup resources after build
         if: ${{ always() }}
         run: |
-          aws ec2 --region ap-southeast-1 describe-instances --filters "Name=tag:testinfra-run-id,Values=${GITHUB_RUN_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -n 1 -I {} aws ec2 terminate-instances --region ap-southeast-1 --instance-ids {} || true
+          aws ec2 --region ap-southeast-1 describe-instances --filters "Name=tag:testinfra-run-id,Values=${GITHUB_RUN_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --region ap-southeast-1 --instance-ids || true
 
       - name: Cleanup AMIs
         if: always()

README.md

@@ -51,7 +51,7 @@ Aside from having [ufw](https://help.ubuntu.com/community/UFW),[fail2ban](https:
 | Goodie | Version | Description |
 | ------------- | :-------------: | ------------- |
 | [PgBouncer](https://www.pgbouncer.org/) | [1.16.1](http://www.pgbouncer.org/changelog.html#pgbouncer-116x) | Set up Connection Pooling. |
-| [PostgREST](https://postgrest.org/en/stable/) | [v10.1.1](https://github.com/PostgREST/postgrest/releases/tag/v10.1.1) | Instantly transform your database into an RESTful API. |
+| [PostgREST](https://postgrest.org/en/stable/) | [v12.2.3](https://github.com/PostgREST/postgrest/releases/tag/v12.2.3) | Instantly transform your database into an RESTful API. |
 | [WAL-G](https://github.com/wal-g/wal-g#wal-g) | [v2.0.1](https://github.com/wal-g/wal-g/releases/tag/v2.0.1) | Tool for physical database backup and recovery. |
 
 ## Install

ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh

@@ -356,7 +356,8 @@ begin
                        end
                      , case when rec.grantee = 'postgres'::regrole then 'supabase_admin'
                             when rec.grantee = 'supabase_admin'::regrole then 'postgres'
-                            else rec.grantee::regrole
+                            when rec.grantee = 0 then 'public'
+                            else rec.grantee::regrole::text
                        end
                      ));
       end if;
@@ -382,7 +383,7 @@ begin
                             when obj->>'objtype' = 'T' then 'types'
                             when obj->>'objtype' = 'n' then 'schemas'
                        end
-                     , rec.grantee::regrole
+                     , case when rec.grantee = 0 then 'public' else rec.grantee::regrole::text end
                      , case when rec.is_grantable then 'with grant option' else '' end
                      ));
       end if;
@@ -529,7 +530,14 @@ $$;
 alter database postgres connection limit -1;
 
 -- #incident-2024-09-12-project-upgrades-are-temporarily-disabled
-grant pg_read_all_data, pg_signal_backend to postgres;
+do $$
+begin
+  if exists (select from pg_authid where rolname = 'pg_read_all_data') then
+    execute('grant pg_read_all_data to postgres');
+  end if;
+end
+$$;
+grant pg_signal_backend to postgres;
 
 set session authorization supabase_admin;
 drop role supabase_tmp;

ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh

@@ -11,6 +11,7 @@
 # them depending on regtypes referencing system OIDs or outdated library files.
 EXTENSIONS_TO_DISABLE=(
     "pg_graphql"
+    "pg_stat_monitor"
 )
 
 PG14_EXTENSIONS_TO_DISABLE=(
@@ -119,20 +120,22 @@ cleanup() {
         CI_start_postgres
     fi
 
+    retry 8 pg_isready -h localhost -U supabase_admin
+
     echo "Re-enabling extensions"
     if [ -f $POST_UPGRADE_EXTENSION_SCRIPT ]; then
-        run_sql -f $POST_UPGRADE_EXTENSION_SCRIPT
+        retry 5 run_sql -f $POST_UPGRADE_EXTENSION_SCRIPT
     fi
 
     echo "Removing SUPERUSER grant from postgres"
-    run_sql -c "ALTER USER postgres WITH NOSUPERUSER;"
+    retry 5 run_sql -c "ALTER USER postgres WITH NOSUPERUSER;"
 
     echo "Resetting postgres database connection limit"
-    run_sql -c "ALTER DATABASE postgres CONNECTION LIMIT -1;"
+    retry 5 run_sql -c "ALTER DATABASE postgres CONNECTION LIMIT -1;"
 
     if [ -z "$IS_CI" ] && [ -z "$IS_LOCAL_UPGRADE" ]; then
         echo "Unmounting data disk from ${MOUNT_POINT}"
-        umount $MOUNT_POINT
+        retry 3 umount $MOUNT_POINT
     fi
     echo "$UPGRADE_STATUS" > /tmp/pg-upgrade-status
 
@@ -208,7 +211,7 @@ function patch_wrappers {
                     WRAPPERS_LIB_PATH_DIR=$(dirname "$WRAPPERS_LIB_PATH")
                     if [ "$WRAPPERS_LIB_PATH" != "$WRAPPERS_LIB_PATH_DIR/${OLD_LIB_FILE_NAME}" ]; then
                         echo "Copying $WRAPPERS_LIB_PATH to $WRAPPERS_LIB_PATH_DIR/${OLD_LIB_FILE_NAME}"
-                        cp "$WRAPPERS_LIB_PATH" "$WRAPPERS_LIB_PATH_DIR/${OLD_LIB_FILE_NAME}"
+                        cp "$WRAPPERS_LIB_PATH" "$WRAPPERS_LIB_PATH_DIR/${OLD_LIB_FILE_NAME}" || true
                     fi
                 fi
             done
@@ -222,7 +225,7 @@ function patch_wrappers {
                     LIB_FILE_NAME=$(basename "$OLD_WRAPPER_LIB_PATH")
                     if [ "$WRAPPERS_LIB_PATH" != "$PGLIBNEW/${LIB_FILE_NAME}" ]; then
                         echo "Copying $WRAPPERS_LIB_PATH to $PGLIBNEW/${LIB_FILE_NAME}"
-                        cp "$WRAPPERS_LIB_PATH" "$PGLIBNEW/${LIB_FILE_NAME}"
+                        cp "$WRAPPERS_LIB_PATH" "$PGLIBNEW/${LIB_FILE_NAME}" || true
                     fi
                 fi
             fi

ansible/files/adminapi.sudoers.conf

@@ -17,6 +17,8 @@ Cmnd_Alias PGBOUNCER = /bin/systemctl start pgbouncer.service, /bin/systemctl st
 %adminapi ALL= NOPASSWD: /usr/bin/systemctl restart postgresql.service
 %adminapi ALL= NOPASSWD: /usr/bin/systemctl show -p NRestarts postgresql.service
 %adminapi ALL= NOPASSWD: /usr/bin/systemctl restart adminapi.service
+%adminapi ALL= NOPASSWD: /usr/bin/systemctl is-active commence-backup.service
+%adminapi ALL= NOPASSWD: /usr/bin/systemctl start commence-backup.service
 %adminapi ALL= NOPASSWD: /bin/systemctl daemon-reload
 %adminapi ALL= NOPASSWD: /bin/systemctl restart services.slice
 %adminapi ALL= NOPASSWD: /usr/sbin/nft -f /etc/nftables/supabase_managed.conf

ansible/files/commence-backup.service.j2

@@ -0,0 +1,12 @@
+[Unit]
+Description=Async commence physical backup
+
+[Service]
+Type=simple
+User=adminapi
+ExecStart=/usr/bin/admin-mgr commence-backup --run-as-service true
+Restart=no
+OOMScoreAdjust=-1000
+
+[Install]
+WantedBy=multi-user.target

ansible/files/envoy_config/lds.yaml

@@ -254,8 +254,10 @@ resources:
                               type.googleapis.com/envoy.extensions.filters.http.rbac.v3.RBACPerRoute
                       - match:
                           safe_regex:
+                            google_re2:
+                              max_program_size: 150
                             regex: >-
-                              /auth/v1/(verify|callback|authorize|sso/saml/(acs|metadata|slo))
+                              /auth/v1/(verify|callback|authorize|sso/saml/(acs|metadata|slo)|\.well-known/(openid-configuration|jwks\.json))
                         route:
                           cluster: gotrue
                           regex_rewrite:

ansible/files/postgresql_config/supautils.conf.j2

@@ -3,7 +3,7 @@ supautils.policy_grants = '{"postgres":["auth.audit_log_entries","auth.identitie
 # full list:                                 address_standardizer, address_standardizer_data_us, adminpack, amcheck, autoinc, bloom, btree_gin, btree_gist, citext, cube, dblink, dict_int, dict_xsyn, earthdistance, file_fdw, fuzzystrmatch, hstore, http, hypopg, index_advisor, insert_username, intagg, intarray, isn, lo, ltree, moddatetime, old_snapshot, orioledb, pageinspect, pg_buffercache, pg_cron, pg_freespacemap, pg_graphql, pg_hashids, pg_jsonschema, pg_net, pg_prewarm, pg_repack, pg_stat_monitor, pg_stat_statements, pg_surgery, pg_tle, pg_trgm, pg_visibility, pg_walinspect, pgaudit, pgcrypto, pgjwt, pgroonga, pgroonga_database, pgrouting, pgrowlocks, pgsodium, pgstattuple, pgtap, plcoffee, pljava, plls, plpgsql, plpgsql_check, plv8, postgis, postgis_raster, postgis_sfcgal, postgis_tiger_geocoder, postgis_topology, postgres_fdw, refint, rum, seg, sslinfo, supabase_vault, supautils, tablefunc, tcn, timescaledb, tsm_system_rows, tsm_system_time, unaccent, uuid-ossp, vector, wrappers, xml2
 # omitted because may be unsafe:             adminpack, amcheck, file_fdw, lo, old_snapshot, pageinspect, pg_buffercache, pg_freespacemap, pg_surgery, pg_visibility
 # omitted because deprecated:                intagg, xml2
-supautils.privileged_extensions = 'address_standardizer, address_standardizer_data_us, autoinc, bloom, btree_gin, btree_gist, citext, cube, dblink, dict_int, dict_xsyn, earthdistance, fuzzystrmatch, hstore, http, hypopg, index_advisor, insert_username, intarray, isn, ltree, moddatetime, orioledb, pg_cron, pg_graphql, pg_hashids, pg_jsonschema, pg_net, pg_repack, pg_stat_monitor, pg_stat_statements, pg_tle, pg_trgm, pg_walinspect, pgaudit, pgcrypto, pgjwt, pg_prewarm, pgroonga, pgroonga_database, pgrouting, pgrowlocks, pgstattuple, pgsodium, pgtap, plcoffee, pljava, plls, plpgsql, plpgsql_check, plv8, postgis, postgis_raster, postgis_sfcgal, postgis_tiger_geocoder, postgis_topology, postgres_fdw, refint, rum, seg, sslinfo, supabase_vault, supautils, tablefunc, tcn, timescaledb, tsm_system_rows, tsm_system_time, unaccent, uuid-ossp, vector, wrappers'
+supautils.privileged_extensions = 'address_standardizer, address_standardizer_data_us, autoinc, bloom, btree_gin, btree_gist, citext, cube, dblink, dict_int, dict_xsyn, earthdistance, fuzzystrmatch, hstore, http, hypopg, index_advisor, insert_username, intarray, isn, ltree, moddatetime, orioledb, pg_cron, pg_graphql, pg_hashids, pg_jsonschema, pg_net, pg_partman, pg_repack, pg_stat_monitor, pg_stat_statements, pg_tle, pg_trgm, pg_walinspect, pgaudit, pgcrypto, pgjwt, pg_prewarm, pgmq, pgroonga, pgroonga_database, pgrouting, pgrowlocks, pgstattuple, pgsodium, pgtap, plcoffee, pljava, plls, plpgsql, plpgsql_check, plv8, postgis, postgis_raster, postgis_sfcgal, postgis_tiger_geocoder, postgis_topology, postgres_fdw, refint, rum, seg, sslinfo, supabase_vault, supautils, tablefunc, tcn, timescaledb, tsm_system_rows, tsm_system_time, unaccent, uuid-ossp, vector, wrappers'
 supautils.privileged_extensions_custom_scripts_path = '/etc/postgresql-custom/extension-custom-scripts'
 supautils.privileged_extensions_superuser = 'supabase_admin'
 supautils.privileged_role = 'postgres'

ansible/tasks/internal/admin-api.yml

@@ -79,6 +79,11 @@
     src: files/adminapi.service.j2
     dest: /etc/systemd/system/adminapi.service
 
+- name: adminapi - create service file for commence backup process
+  template:
+     src: files/commence-backup.service.j2
+     dest: /etc/systemd/system/commence-backup.service
+
 - name: UFW - Allow connections to adminapi ports
   ufw:
     rule: allow

ansible/vars.yml

@@ -17,8 +17,8 @@ postgrest_release: "12.2.3"
 postgrest_arm_release_checksum: sha1:fbfd6613d711ce1afa25c42d5df8f1b017f396f9
 postgrest_x86_release_checksum: sha1:61c513f91a8931be4062587b9d4a18b42acf5c05
 
-gotrue_release: 2.160.0
-gotrue_release_checksum: sha1:391b3f174e3d82cc806b2ba8d65253b7b2c874a6
+gotrue_release: 2.162.0
+gotrue_release_checksum: sha1:855b23bd002577290c7d42d7042ac0f5316984b1
 
 aws_cli_release: "2.2.7"
 
@@ -50,8 +50,8 @@ postgres_exporter_release_checksum:
   arm64: sha256:29ba62d538b92d39952afe12ee2e1f4401250d678ff4b354ff2752f4321c87a0
   amd64: sha256:cb89fc5bf4485fb554e0d640d9684fae143a4b2d5fa443009bd29c59f9129e84
 
-adminapi_release: 0.68.0
-adminmgr_release: 0.22.1
+adminapi_release: 0.71.1
+adminmgr_release: 0.24.0
 
 # Postgres Extensions
 postgis_release: "3.3.2"

common-nix.vars.pkr.hcl

@@ -1 +1 @@
-postgres-version = "15.6.1.123"
+postgres-version = "15.8.1.003"

common.vars.pkr.hcl

@@ -1 +1 @@
-postgres-version = "15.1.1.93"
+postgres-version = "15.1.1.94"

docker/all-in-one/entrypoint.sh

@@ -101,7 +101,7 @@ function setup_postgres {
   mv /etc/postgresql.schema.sql /docker-entrypoint-initdb.d/migrations/99-schema.sql
 
   tar -xzvf "$INIT_PAYLOAD_PATH" -C / ./etc/postgresql-custom/pgsodium_root.key
-  echo "include = '/etc/postgresql-custom/postgresql-platform-defaults.conf'" >>$PG_CONF
+  sed -i "/# Automatically generated optimizations/i # Supabase Platform Defaults\ninclude = '/etc/postgresql-custom/platform-defaults.conf'\n" $PG_CONF
 
   # TODO (darora): walg enablement is temporarily performed here until changes from https://github.com/supabase/postgres/pull/639 get picked up
   # other things will still be needed in the future (auth_delay config)

docker/all-in-one/etc/adminapi/adminapi.yaml

@@ -42,6 +42,13 @@ upstream_metrics_sources:
         value: {{ .ProjectRef }}
       - name: service_type
         value: gotrue
+  - name: postgrest
+    url: "http://localhost:3001/metrics"
+    labels_to_attach:
+      - name: supabase_project_ref
+        value: {{ .ProjectRef }}
+      - name: service_type
+        value: postgrest
 monitoring:
   disk_usage:
     enabled: true

docker/all-in-one/etc/kong/kong.yml

@@ -27,6 +27,12 @@ services:
       routes: [{ name: auth-v1-open-saml, strip_path: true, paths: [/auth/v1/sso/saml/] }],
       plugins: [{ name: cors }],
     }
+  - {
+      name: auth-v1-open-well-known,
+      url: 'http://localhost:9999/.well-known/',
+      routes: [{ name: auth-v1-open-well-known, strip_path: true, paths: [/auth/v1/.well-known/] }],
+      plugins: [{ name: cors }],
+    }
   - {
       name: auth-v1,
       url: 'http://localhost:9999/',

docker/all-in-one/etc/postgresql-custom/postgresql-platform-defaults.conf

@@ -1,8 +1,6 @@
 # these get imported _after_ the user specified overrides
 row_security = on
 wal_level = logical
-max_wal_senders = 10
-max_replication_slots = 5
 log_connections = on
 statement_timeout = 120000
 jit = off

flake.nix

@@ -38,7 +38,6 @@
             # want to have an arbitrary order, since it might matter. being
             # explicit is better.
             (import ./nix/overlays/cargo-pgrx.nix)
-            (import ./nix/overlays/gdal-small.nix)
             (import ./nix/overlays/psql_16-oriole.nix)
 
           ];
@@ -68,15 +67,9 @@
               };
             })
             (import ./nix/overlays/cargo-pgrx-0-11-3.nix)
-            # (import ./nix/overlays/postgis.nix)
-            #(import ./nix/overlays/gdal-small.nix)
-
           ];
         };
-        postgresql_15 = pkgs.postgresql.postgresql_15;
-        postgresql = pkgs.postgresql.postgresql_15;
         sfcgal = pkgs.callPackage ./nix/ext/sfcgal/sfcgal.nix { };
-        pg_regress = pkgs.callPackage ./nix/ext/pg_regress.nix { inherit postgresql; };
         supabase-groonga = pkgs.callPackage ./nix/supabase-groonga.nix { };
         mecab-naist-jdic = pkgs.callPackage ./nix/ext/mecab-naist-jdic/default.nix { };
         # Our list of PostgreSQL extensions which come from upstream Nixpkgs.
@@ -110,6 +103,7 @@
           ./nix/ext/pgroonga.nix
           ./nix/ext/index_advisor.nix
           ./nix/ext/wal2json.nix
+          ./nix/ext/pgmq.nix
           ./nix/ext/pg_repack.nix
           ./nix/ext/pg-safeupdate.nix
           ./nix/ext/plpgsql-check.nix
@@ -126,6 +120,7 @@
           ./nix/ext/pg_hashids.nix
           ./nix/ext/pgsodium.nix
           ./nix/ext/pg_graphql.nix
+          ./nix/ext/pg_partman.nix
           ./nix/ext/pg_stat_monitor.nix
           ./nix/ext/pg_jsonschema.nix
           ./nix/ext/pgvector.nix
@@ -284,22 +279,49 @@
         # be used with 'nix build'. Don't use the names listed below; check the
         # name in 'nix flake show' in order to make sure exactly what name you
         # want.
-        basePackages = {
+        basePackages = let
+          # Function to get the PostgreSQL version from the attribute name
+          getVersion = name: 
+            let
+              match = builtins.match "psql_([0-9]+)" name;
+            in
+            if match == null then null else builtins.head match;
+
+          # Define the available PostgreSQL versions
+          postgresVersions = {
+            psql_15 = makePostgres "15";
+            # Uncomment the line below to enable PostgreSQL 16
+            # psql_16 = makePostgres "16";
+            # psql_orioledb_16 = makeOrioleDbPostgres "16_23" postgresql_orioledb_16;
+          };
+
+          # Find the active PostgreSQL version
+          activeVersion = getVersion (builtins.head (builtins.attrNames postgresVersions));
+
+          # Function to create the pg_regress package
+          makePgRegress = version:
+            let
+              postgresqlPackage = pkgs."postgresql_${version}";
+            in
+              pkgs.callPackage ./nix/ext/pg_regress.nix { 
+                postgresql = postgresqlPackage;
+              };
+          postgresql_15 = getPostgresqlPackage "15";
+        in 
+        postgresVersions //{
           supabase-groonga = supabase-groonga;
           # PostgreSQL versions.
           psql_15 = makePostgres "15";
-          #psql_16 = makePostgres "16";
           #psql_orioledb_16 = makeOrioleDbPostgres "16_23" postgresql_orioledb_16;
           sfcgal = sfcgal;
-          pg_regress = pg_regress;
           pg_prove = pkgs.perlPackages.TAPParserSourceHandlerpgTAP;
-          postgresql_15 = pkgs.postgresql_15;
-
+          inherit postgresql_15;
+          postgresql_15_debug = if pkgs.stdenv.isLinux then postgresql_15.debug else null;
           postgresql_15_src = pkgs.stdenv.mkDerivation {
             pname = "postgresql-15-src";
-            version = pkgs.postgresql_15.version;
+            version = postgresql_15.version;
 
-            src = pkgs.postgresql_15.src;
+            src = postgresql_15.src;
 
             nativeBuildInputs = [ pkgs.bzip2 ];
 
@@ -319,6 +341,7 @@
           };
           mecab_naist_jdic = mecab-naist-jdic;
           supabase_groonga = supabase-groonga;
+          pg_regress = makePgRegress activeVersion;
           # Start a version of the server.
           start-server =
             let
@@ -455,6 +478,7 @@
             sqlTests = ./nix/tests/smoke;
             pg_prove = pkgs.perlPackages.TAPParserSourceHandlerpgTAP;
             supabase-groonga = pkgs.callPackage ./nix/supabase-groonga.nix { };
+            pg_regress = basePackages.pg_regress;
           in
           pkgs.runCommand "postgres-${pgpkg.version}-check-harness"
             {

migrations/tests/extensions/01-postgis.sql

@@ -19,7 +19,7 @@ grant all privileges on all sequences in schema tiger, tiger_data to postgres wi
 alter default privileges in schema tiger, tiger_data grant all on tables to postgres with grant option;
 alter default privileges in schema tiger, tiger_data grant all on routines to postgres with grant option;
 alter default privileges in schema tiger, tiger_data grant all on sequences to postgres with grant option;
-
+SET search_path TO extensions, public, tiger, tiger_data;
 -- postgres role should have access
 set local role postgres;
 select tiger.pprint_addy(tiger.pagc_normalize_address('710 E Ben White Blvd, Austin, TX 78704'));

nix/ext/0001-build-Allow-using-V8-from-system.patch

@@ -5,7 +5,7 @@ index 38879cc..6e78eeb 100644
 @@ -20,6 +20,7 @@ OBJS = $(SRCS:.cc=.o)
  MODULE_big = plv8-$(PLV8_VERSION)
  EXTENSION = plv8
- PLV8_DATA = plv8.control plv8--$(PLV8_VERSION).sql $(wildcard upgrade/*.sql)
+ PLV8_DATA = plv8.control plv8--$(PLV8_VERSION).sql
 +USE_SYSTEM_V8 = 0