migrate dbt-postgres into the monorepo

dbt-labs · Jan 23, 2025 · cae11b6 · cae11b6
2 parents 8a8aa48 + 49141a6
commit cae11b6
Show file tree

Hide file tree

Showing 259 changed files with 20,380 additions and 2 deletions.
diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml
@@ -37,6 +37,8 @@ body:
             required: false
         -   label: dbt-bigquery
             required: false
+        -   label: dbt-postgres
+            required: false
         -   label: dbt-redshift
             required: false
         -   label: dbt-spark

diff --git a/.github/ISSUE_TEMPLATE/regression-report.yml b/.github/ISSUE_TEMPLATE/regression-report.yml
@@ -32,6 +32,8 @@ body:
             required: false
         -   label: dbt-bigquery
             required: false
+        -   label: dbt-postgres
+            required: false
         -   label: dbt-redshift
             required: false
         -   label: dbt-spark

diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -7,6 +7,7 @@ updates:
     -   "/dbt-athena"
     -   "/dbt-athena-community"
     -   "/dbt-bigquery"
+    -   "/dbt-postgres"
     -   "/dbt-redshift"
     -   "/dbt-spark"
     schedule:
@@ -19,6 +20,7 @@ updates:
 -   package-ecosystem: "docker"
     directories:
     -   "/dbt-bigquery/docker"
+    -   "/dbt-postgres/docker"
     -   "/dbt-redshift/docker"
     -   "/dbt-spark/docker"
     schedule:

diff --git a/.github/workflows/_generate-changelog.yml b/.github/workflows/_generate-changelog.yml
@@ -36,6 +36,7 @@ on:
                 -   "dbt-athena"
                 -   "dbt-athena-community"
                 -   "dbt-bigquery"
+                -   "dbt-postgres"
                 -   "dbt-redshift"
                 -   "dbt-spark"
             merge:

diff --git a/.github/workflows/_integration-tests.yml b/.github/workflows/_integration-tests.yml
@@ -32,6 +32,7 @@ on:
                 -   "dbt-athena"
                 -   "dbt-athena-community"
                 -   "dbt-bigquery"
+                -   "dbt-postgres"
                 -   "dbt-redshift"
                 -   "dbt-spark"
             branch:
@@ -128,6 +129,85 @@ jobs:
             if: ${{ inputs.python-version == '3.9' }}  # we only run this for one version to run in series
             working-directory: ./${{ inputs.package }}
 
+    integration-tests-postgres:
+        if: inputs.package == 'dbt-postgres'
+        runs-on: ${{ inputs.os }}
+        environment:
+            name: ${{ inputs.package }}
+        env:
+            POSTGRES_TEST_HOST: ${{ vars.POSTGRES_TEST_HOST }}
+            POSTGRES_TEST_PORT: ${{ vars.POSTGRES_TEST_PORT }}
+            POSTGRES_TEST_USER: ${{ vars.POSTGRES_TEST_USER }}
+            POSTGRES_TEST_PASS: ${{ secrets.POSTGRES_TEST_PASS }}
+            POSTGRES_TEST_DATABASE: ${{ vars.POSTGRES_TEST_DATABASE }}
+            POSTGRES_TEST_THREADS: ${{ vars.POSTGRES_TEST_THREADS }}
+        services:
+            postgres:
+                image: postgres
+                env:
+                    POSTGRES_PASSWORD: postgres
+                options: >-
+                    --health-cmd pg_isready
+                    --health-interval 10s
+                    --health-timeout 5s
+                    --health-retries 5
+                ports:
+                -   ${{ vars.POSTGRES_TEST_PORT }}:5432
+        steps:
+        -   uses: actions/checkout@v4
+            with:
+                ref: ${{ inputs.branch }}
+                repository: ${{ inputs.repository }}
+        -   uses: actions/setup-python@v5
+            with:
+                python-version: ${{ inputs.python-version }}
+        -   run: psql -f ./scripts/setup_test_database.sql
+            working-directory: ./dbt-postgres
+            env:
+                PGHOST: ${{ vars.POSTGRES_TEST_HOST }}
+                PGPORT: ${{ vars.POSTGRES_TEST_PORT }}
+                PGUSER: postgres
+                PGPASSWORD: postgres
+                PGDATABASE: postgres
+        -   uses: pypa/hatch@install
+        -   run: hatch run integration-tests tests/functional --ddtrace
+            working-directory: ./${{ inputs.package }}
+
+    integration-tests-postgres-psycopg2-check:
+        if: inputs.package == 'dbt-postgres'
+        runs-on: ${{ inputs.os }}
+        steps:
+        -   uses: actions/checkout@v4
+        -   uses: actions/setup-python@v5
+            with:
+                python-version: ${{ inputs.python-version }}
+        -   if: startsWith(inputs.os, 'macos')
+            run: brew install postgresql
+        -   name: "Test psycopg2 - default"
+            run: scripts/psycopg2-check.sh
+            working-directory: ./${{ inputs.package }}
+            env:
+                PSYCOPG2_EXPECTED_NAME: psycopg2-binary
+        -   name: "Test psycopg2 - invalid override"
+            run: scripts/psycopg2-check.sh
+            working-directory: ./${{ inputs.package }}
+            env:
+                DBT_PSYCOPG2_NAME: rubber-baby-buggy-bumpers
+                PSYCOPG2_EXPECTED_NAME: psycopg2-binary
+        -   name: "Test psycopg2 - override"
+            run: scripts/psycopg2-check.sh
+            working-directory: ./${{ inputs.package }}
+            env:
+                DBT_PSYCOPG2_NAME: psycopg2
+                PSYCOPG2_EXPECTED_NAME: psycopg2-binary  # we have not implemented the hook yet, so this doesn't work
+        -   name: "Test psycopg2 - manual override"
+            # verify that the workaround documented in the `README.md` continues to work
+            run: scripts/psycopg2-check.sh
+            working-directory: ./${{ inputs.package }}
+            env:
+                PSYCOPG2_WORKAROUND: true
+                PSYCOPG2_EXPECTED_NAME: psycopg2
+
     integration-tests-redshift:
         if: ${{ inputs.package == 'dbt-redshift' }}
         runs-on: ${{ inputs.os }}

diff --git a/.github/workflows/_publish-internal.yml b/.github/workflows/_publish-internal.yml
@@ -24,6 +24,7 @@ on:
                 -   "dbt-adapters"
                 -   "dbt-athena"
                 -   "dbt-bigquery"
+                -   "dbt-postgres"
                 -   "dbt-redshift"
                 -   "dbt-spark"
             deploy-to:

diff --git a/.github/workflows/_publish-pypi.yml b/.github/workflows/_publish-pypi.yml
@@ -25,6 +25,7 @@ on:
                 -   "dbt-athena"
                 -   "dbt-athena-community"
                 -   "dbt-bigquery"
+                -   "dbt-postgres"
                 -   "dbt-redshift"
                 -   "dbt-spark"
             deploy-to:

diff --git a/.github/workflows/_unit-tests.yml b/.github/workflows/_unit-tests.yml
@@ -33,6 +33,7 @@ on:
                 -   "dbt-athena"
                 -   "dbt-athena-community"
                 -   "dbt-bigquery"
+                -   "dbt-postgres"
                 -   "dbt-redshift"
                 -   "dbt-spark"
             branch:

diff --git a/.github/workflows/_verify-build.yml b/.github/workflows/_verify-build.yml
@@ -34,6 +34,7 @@ on:
                 -   "dbt-athena"
                 -   "dbt-athena-community"
                 -   "dbt-bigquery"
+                -   "dbt-postgres"
                 -   "dbt-redshift"
                 -   "dbt-spark"
             branch:

diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
@@ -13,6 +13,7 @@ on:
                 -   "dbt-athena"
                 -   "dbt-athena-community"
                 -   "dbt-bigquery"
+                -   "dbt-postgres"
                 -   "dbt-redshift"
                 -   "dbt-spark"
             deploy-to:
@@ -56,7 +57,7 @@ jobs:
     generate-changelog:
         needs: [unit-tests, integration-tests]
         if: |
-          always() && 
+          always() &&
           (needs.integration-tests.result == 'skipped' || needs.integration-tests.result == 'success') &&
           (needs.unit-tests.result == 'skipped' || needs.unit-tests.result == 'success')
         uses: ./.github/workflows/_generate-changelog.yml

diff --git a/.github/workflows/pull-request-checks.yml b/.github/workflows/pull-request-checks.yml
@@ -35,6 +35,7 @@ jobs:
                 -   "dbt-athena"
                 -   "dbt-athena-community"
                 -   "dbt-bigquery"
+                -   "dbt-postgres"
                 -   "dbt-redshift"
                 -   "dbt-spark"
                 os: [ubuntu-22.04]
@@ -56,6 +57,7 @@ jobs:
                 -   "dbt-athena"
                 -   "dbt-athena-community"
                 -   "dbt-bigquery"
+                -   "dbt-postgres"
                 -   "dbt-redshift"
                 -   "dbt-spark"
                 os: [ ubuntu-22.04 ]
@@ -76,6 +78,7 @@ jobs:
                 -   "dbt-athena"
                 -   "dbt-athena-community"
                 -   "dbt-bigquery"
+                -   "dbt-postgres"
                 -   "dbt-redshift"
                 -   "dbt-spark"
                 os: [ubuntu-22.04]

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -34,7 +34,7 @@ repos:
     rev: 7.1.1
     hooks:
     -   id: flake8
-        exclude: dbt/adapters/events/adapter_types_pb2.py|tests/functional/|dbt-spark/tests/|dbt-bigquery/tests/|dbt-redshift/tests
+        exclude: dbt/adapters/events/adapter_types_pb2.py|tests/functional/|dbt-spark/tests/|dbt-bigquery/tests/|dbt-redshift/tests|dbt-postgres/tests
         args:
         -   --max-line-length=99
         -   --select=E,F,W

diff --git a/dbt-postgres/.changes/0.0.0.md b/dbt-postgres/.changes/0.0.0.md
@@ -0,0 +1,3 @@
+## Previous Releases
+For information on prior major and minor releases, see their changelogs:
+- [1.8](https://github.com/dbt-labs/dbt-postgres/blob/1.8.latest/CHANGELOG.md)
diff --git a/dbt-postgres/.changes/1.9.0-b1.md b/dbt-postgres/.changes/1.9.0-b1.md
diff --git a/dbt-postgres/.changes/1.9.0.md b/dbt-postgres/.changes/1.9.0.md
@@ -0,0 +1,35 @@
+## dbt-postgres 1.9.0 - December 09, 2024
+
+### Breaking Changes
+
+- Drop support for Python 3.8 ([#161](https://github.com/dbt-labs/dbt-postgres/issues/161))
+
+### Features
+
+- Add tests for cross-database `cast` macro ([#76](https://github.com/dbt-labs/dbt-postgres/issues/76))
+- Cross-database `date` macro ([#82](https://github.com/dbt-labs/dbt-postgres/issues/82))
+- Add support for Python 3.12 ([#17](https://github.com/dbt-labs/dbt-postgres/issues/17))
+- Allow configuring snapshot column names ([#144](https://github.com/dbt-labs/dbt-postgres/issues/144))
+- Microbatch incremental strategy implementation: merge ([#149](https://github.com/dbt-labs/dbt-postgres/issues/149))
+- Enable setting current value of dbt_valid_to ([#151](https://github.com/dbt-labs/dbt-postgres/issues/151))
+
+### Fixes
+
+- Fix the semicolon semantics for indexes while respecting other bug fix ([#85](https://github.com/dbt-labs/dbt-postgres/issues/85))
+- Default to psycopg2-binary and allow overriding to psycopg2 via DBT_PSYCOPG2_NAME (restores previous behavior) ([#96](https://github.com/dbt-labs/dbt-postgres/issues/96))
+- Fix `persist_docs` for `materialized_view` materializations. Previously, using this configuration with materialized view models would lead to an error. ([#120](https://github.com/dbt-labs/dbt-postgres/issues/120))
+
+### Under the Hood
+
+- Add support for experimental record/replay testing. ([#123](https://github.com/dbt-labs/dbt-postgres/issues/123))
+- Updating changie.yaml to add contributors and PR links ([#109](https://github.com/dbt-labs/dbt-postgres/issues/109))
+
+### Contributors
+- [@dbeatty10](https://github.com/dbeatty10) ([#76](https://github.com/dbt-labs/dbt-postgres/issues/76), [#82](https://github.com/dbt-labs/dbt-postgres/issues/82))
+- [@gshank](https://github.com/gshank) ([#144](https://github.com/dbt-labs/dbt-postgres/issues/144), [#151](https://github.com/dbt-labs/dbt-postgres/issues/151))
+- [@leahwicz](https://github.com/leahwicz) ([#109](https://github.com/dbt-labs/dbt-postgres/issues/109))
+- [@michelleark](https://github.com/michelleark) ([#149](https://github.com/dbt-labs/dbt-postgres/issues/149))
+- [@mikealfare](https://github.com/mikealfare) ([#161](https://github.com/dbt-labs/dbt-postgres/issues/161), [#17](https://github.com/dbt-labs/dbt-postgres/issues/17), [#96](https://github.com/dbt-labs/dbt-postgres/issues/96))
+- [@morsapaes](https://github.com/morsapaes) ([#120](https://github.com/dbt-labs/dbt-postgres/issues/120))
+- [@peterallenwebb](https://github.com/peterallenwebb) ([#123](https://github.com/dbt-labs/dbt-postgres/issues/123))
+- [@versusfacit](https://github.com/versusfacit) ([#85](https://github.com/dbt-labs/dbt-postgres/issues/85))
diff --git a/dbt-postgres/.changes/header.tpl.md b/dbt-postgres/.changes/header.tpl.md
@@ -0,0 +1,6 @@
+# Changelog
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html),
+and is generated by [Changie](https://github.com/miniscruff/changie).
diff --git a/dbt-postgres/.changes/unreleased/.gitkeep b/dbt-postgres/.changes/unreleased/.gitkeep
diff --git a/dbt-postgres/.changes/unreleased/Under the Hood-20241201-122244.yaml b/dbt-postgres/.changes/unreleased/Under the Hood-20241201-122244.yaml
@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Migrate to a monorepo in dbt-adapters
+time: 2024-12-01T12:22:44.407931-05:00
+custom:
+  Author: mikealfare
+  Issue: "176"