Merge branch 'main' into ADAP-1167/add-athena-adapter-telemetry

dbt-labs · Jan 27, 2025 · f62419e · f62419e
2 parents 8451a92 + aa29250
commit f62419e
Show file tree

Hide file tree

Showing 443 changed files with 32,869 additions and 27 deletions.
diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml
@@ -37,8 +37,12 @@ body:
             required: false
         -   label: dbt-bigquery
             required: false
+        -   label: dbt-postgres
+            required: false
         -   label: dbt-redshift
             required: false
+        -   label: dbt-snowflake
+            required: false
         -   label: dbt-spark
             required: false
 -   type: textarea

diff --git a/.github/ISSUE_TEMPLATE/regression-report.yml b/.github/ISSUE_TEMPLATE/regression-report.yml
@@ -32,8 +32,12 @@ body:
             required: false
         -   label: dbt-bigquery
             required: false
+        -   label: dbt-postgres
+            required: false
         -   label: dbt-redshift
             required: false
+        -   label: dbt-snowflake
+            required: false
         -   label: dbt-spark
             required: false
 -   type: textarea

diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -7,7 +7,9 @@ updates:
     -   "/dbt-athena"
     -   "/dbt-athena-community"
     -   "/dbt-bigquery"
+    -   "/dbt-postgres"
     -   "/dbt-redshift"
+    -   "/dbt-snowflake"
     -   "/dbt-spark"
     schedule:
         interval: "daily"
@@ -19,7 +21,9 @@ updates:
 -   package-ecosystem: "docker"
     directories:
     -   "/dbt-bigquery/docker"
+    -   "/dbt-postgres/docker"
     -   "/dbt-redshift/docker"
+    -   "/dbt-snowflake/docker"
     -   "/dbt-spark/docker"
     schedule:
         interval: "weekly"

diff --git a/.github/workflows/_generate-changelog.yml b/.github/workflows/_generate-changelog.yml
@@ -36,7 +36,9 @@ on:
                 -   "dbt-athena"
                 -   "dbt-athena-community"
                 -   "dbt-bigquery"
+                -   "dbt-postgres"
                 -   "dbt-redshift"
+                -   "dbt-snowflake"
                 -   "dbt-spark"
             merge:
                 description: "Choose whether to merge the changelog branch"

diff --git a/.github/workflows/_integration-tests.yml b/.github/workflows/_integration-tests.yml
@@ -32,7 +32,9 @@ on:
                 -   "dbt-athena"
                 -   "dbt-athena-community"
                 -   "dbt-bigquery"
+                -   "dbt-postgres"
                 -   "dbt-redshift"
+                -   "dbt-snowflake"
                 -   "dbt-spark"
             branch:
                 description: "Choose the branch to test"
@@ -128,6 +130,85 @@ jobs:
             if: ${{ inputs.python-version == '3.9' }}  # we only run this for one version to run in series
             working-directory: ./${{ inputs.package }}
 
+    integration-tests-postgres:
+        if: inputs.package == 'dbt-postgres'
+        runs-on: ${{ inputs.os }}
+        environment:
+            name: ${{ inputs.package }}
+        env:
+            POSTGRES_TEST_HOST: ${{ vars.POSTGRES_TEST_HOST }}
+            POSTGRES_TEST_PORT: ${{ vars.POSTGRES_TEST_PORT }}
+            POSTGRES_TEST_USER: ${{ vars.POSTGRES_TEST_USER }}
+            POSTGRES_TEST_PASS: ${{ secrets.POSTGRES_TEST_PASS }}
+            POSTGRES_TEST_DATABASE: ${{ vars.POSTGRES_TEST_DATABASE }}
+            POSTGRES_TEST_THREADS: ${{ vars.POSTGRES_TEST_THREADS }}
+        services:
+            postgres:
+                image: postgres
+                env:
+                    POSTGRES_PASSWORD: postgres
+                options: >-
+                    --health-cmd pg_isready
+                    --health-interval 10s
+                    --health-timeout 5s
+                    --health-retries 5
+                ports:
+                -   ${{ vars.POSTGRES_TEST_PORT }}:5432
+        steps:
+        -   uses: actions/checkout@v4
+            with:
+                ref: ${{ inputs.branch }}
+                repository: ${{ inputs.repository }}
+        -   uses: actions/setup-python@v5
+            with:
+                python-version: ${{ inputs.python-version }}
+        -   run: psql -f ./scripts/setup_test_database.sql
+            working-directory: ./dbt-postgres
+            env:
+                PGHOST: ${{ vars.POSTGRES_TEST_HOST }}
+                PGPORT: ${{ vars.POSTGRES_TEST_PORT }}
+                PGUSER: postgres
+                PGPASSWORD: postgres
+                PGDATABASE: postgres
+        -   uses: pypa/hatch@install
+        -   run: hatch run integration-tests tests/functional --ddtrace
+            working-directory: ./${{ inputs.package }}
+
+    integration-tests-postgres-psycopg2-check:
+        if: inputs.package == 'dbt-postgres'
+        runs-on: ${{ inputs.os }}
+        steps:
+        -   uses: actions/checkout@v4
+        -   uses: actions/setup-python@v5
+            with:
+                python-version: ${{ inputs.python-version }}
+        -   if: startsWith(inputs.os, 'macos')
+            run: brew install postgresql
+        -   name: "Test psycopg2 - default"
+            run: scripts/psycopg2-check.sh
+            working-directory: ./${{ inputs.package }}
+            env:
+                PSYCOPG2_EXPECTED_NAME: psycopg2-binary
+        -   name: "Test psycopg2 - invalid override"
+            run: scripts/psycopg2-check.sh
+            working-directory: ./${{ inputs.package }}
+            env:
+                DBT_PSYCOPG2_NAME: rubber-baby-buggy-bumpers
+                PSYCOPG2_EXPECTED_NAME: psycopg2-binary
+        -   name: "Test psycopg2 - override"
+            run: scripts/psycopg2-check.sh
+            working-directory: ./${{ inputs.package }}
+            env:
+                DBT_PSYCOPG2_NAME: psycopg2
+                PSYCOPG2_EXPECTED_NAME: psycopg2-binary  # we have not implemented the hook yet, so this doesn't work
+        -   name: "Test psycopg2 - manual override"
+            # verify that the workaround documented in the `README.md` continues to work
+            run: scripts/psycopg2-check.sh
+            working-directory: ./${{ inputs.package }}
+            env:
+                PSYCOPG2_WORKAROUND: true
+                PSYCOPG2_EXPECTED_NAME: psycopg2
+
     integration-tests-redshift:
         if: ${{ inputs.package == 'dbt-redshift' }}
         runs-on: ${{ inputs.os }}
@@ -187,6 +268,38 @@ jobs:
         -   run: hatch run integration-tests tests/functional -m flaky -n1 --ddtrace
             working-directory: ./${{ inputs.package }}
 
+    integration-tests-snowflake:
+        if: ${{ inputs.package == 'dbt-snowflake' }}
+        runs-on: ${{ inputs.os }}
+        environment:
+            name: ${{ inputs.package }}
+        env:
+            SNOWFLAKE_TEST_ACCOUNT: ${{ secrets.SNOWFLAKE_TEST_ACCOUNT }}
+            SNOWFLAKE_TEST_USER: ${{ vars.SNOWFLAKE_TEST_USER }}
+            SNOWFLAKE_TEST_PASSWORD: ${{ secrets.SNOWFLAKE_TEST_PASSWORD }}
+            SNOWFLAKE_TEST_ROLE: ${{ vars.SNOWFLAKE_TEST_ROLE }}
+            SNOWFLAKE_TEST_DATABASE: ${{ vars.SNOWFLAKE_TEST_DATABASE }}
+            SNOWFLAKE_TEST_WAREHOUSE: ${{ vars.SNOWFLAKE_TEST_WAREHOUSE }}
+            SNOWFLAKE_TEST_ALT_DATABASE: ${{ vars.SNOWFLAKE_TEST_ALT_DATABASE }}
+            SNOWFLAKE_TEST_ALT_WAREHOUSE: ${{ vars.SNOWFLAKE_TEST_ALT_WAREHOUSE }}
+            SNOWFLAKE_TEST_QUOTED_DATABASE: ${{ vars.SNOWFLAKE_TEST_QUOTED_DATABASE }}
+            SNOWFLAKE_TEST_OAUTH_CLIENT_ID: ${{ vars.SNOWFLAKE_TEST_OAUTH_CLIENT_ID }}
+            SNOWFLAKE_TEST_OAUTH_CLIENT_SECRET: ${{ secrets.SNOWFLAKE_TEST_OAUTH_CLIENT_SECRET }}
+            SNOWFLAKE_TEST_OAUTH_REFRESH_TOKEN: ${{ secrets.SNOWFLAKE_TEST_OAUTH_REFRESH_TOKEN }}
+            SNOWFLAKE_TEST_PRIVATE_KEY: ${{ secrets.SNOWFLAKE_TEST_PRIVATE_KEY }}
+            SNOWFLAKE_TEST_PRIVATE_KEY_PASSPHRASE: ${{ secrets.SNOWFLAKE_TEST_PRIVATE_KEY_PASSPHRASE }}
+        steps:
+        -   uses: actions/checkout@v4
+            with:
+                ref: ${{ inputs.branch }}
+                repository: ${{ inputs.repository }}
+        -   uses: actions/setup-python@v5
+            with:
+                python-version: ${{ inputs.python-version }}
+        -   uses: pypa/hatch@install
+        -   run: hatch run integration-tests tests/functional --ddtrace
+            working-directory: ./${{ inputs.package }}
+
     integration-tests-spark:
         if: ${{ inputs.package == 'dbt-spark' }}
         runs-on: ${{ inputs.os }}

diff --git a/.github/workflows/_publish-internal.yml b/.github/workflows/_publish-internal.yml
@@ -24,7 +24,9 @@ on:
                 -   "dbt-adapters"
                 -   "dbt-athena"
                 -   "dbt-bigquery"
+                -   "dbt-postgres"
                 -   "dbt-redshift"
+                -   "dbt-snowflake"
                 -   "dbt-spark"
             deploy-to:
                 description: "Choose whether to publish to test or prod"
@@ -39,6 +41,9 @@ defaults:
     run:
         shell: bash
 
+env:
+  TEMP_PROFILE_NAME: "temp_aws_profile"
+
 jobs:
     package:
         uses: ./.github/workflows/_package-directory.yml
@@ -58,11 +63,15 @@ jobs:
             with:
                 python-version: ${{ vars.DEFAULT_PYTHON_VERSION }}
         -   uses: pypa/hatch@install
-        -   uses: aws-actions/configure-aws-credentials@v4
-            with:
-                aws-region: ${{ vars.AWS_REGION }}
-                aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-                aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        -   name: "Configure AWS profile for upload"
+            run: |
+              aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID }} --profile ${{ env.TEMP_PROFILE_NAME }}
+              aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY }} --profile ${{ env.TEMP_PROFILE_NAME }}
+              aws configure set region ${{ vars.AWS_REGION }} --profile ${{ env.TEMP_PROFILE_NAME }}
+              aws configure set output text --profile ${{ env.TEMP_PROFILE_NAME }}
+              aws codeartifact login --tool twine --repository ${{ vars.AWS_REPOSITORY }} \
+              --domain ${{ vars.AWS_DOMAIN }} --domain-owner ${{ secrets.AWS_DOMAIN_OWNER }} \
+              --region ${{ vars.AWS_REGION }} --profile ${{ env.TEMP_PROFILE_NAME }}
         -   id: package
             run: |
                 # strip the pre-release off to find all iterations of this patch
@@ -91,21 +100,8 @@ jobs:
         -   run: sed -i "/dbt-core[<>~=]/d" ./pyproject.toml
             working-directory: ./${{ needs.package.outputs.directory }}
         -   run: |
-                export HATCH_INDEX_USER=${{ secrets.AWS_USER }}
-
-                export HATCH_INDEX_AUTH=$(aws codeartifact get-authorization-token \
-                    --domain ${{ vars.AWS_DOMAIN }} \
-                    --output text \
-                    --query authorizationToken)
-
-                export HATCH_INDEX_REPO=$(aws codeartifact get-repository-endpoint \
-                    --domain ${{ vars.AWS_DOMAIN }} \
-                    --repository ${{ vars.AWS_REPOSITORY }} \
-                    --format pypi \
-                    --output text \
-                    --query repositoryEndpoint)
-
+                pip install twine
                 hatch build --clean
                 hatch run build:check-all
-                hatch publish
+                twine upload --repository codeartifact dist/*
             working-directory: ./${{ needs.package.outputs.directory }}
diff --git a/.github/workflows/_publish-pypi.yml b/.github/workflows/_publish-pypi.yml
@@ -25,7 +25,9 @@ on:
                 -   "dbt-athena"
                 -   "dbt-athena-community"
                 -   "dbt-bigquery"
+                -   "dbt-postgres"
                 -   "dbt-redshift"
+                -   "dbt-snowflake"
                 -   "dbt-spark"
             deploy-to:
                 description: "Choose whether to publish to test or prod"

diff --git a/.github/workflows/_unit-tests.yml b/.github/workflows/_unit-tests.yml
@@ -33,7 +33,9 @@ on:
                 -   "dbt-athena"
                 -   "dbt-athena-community"
                 -   "dbt-bigquery"
+                -   "dbt-postgres"
                 -   "dbt-redshift"
+                -   "dbt-snowflake"
                 -   "dbt-spark"
             branch:
                 description: "Choose the branch to test"

diff --git a/.github/workflows/_verify-build.yml b/.github/workflows/_verify-build.yml
@@ -34,7 +34,9 @@ on:
                 -   "dbt-athena"
                 -   "dbt-athena-community"
                 -   "dbt-bigquery"
+                -   "dbt-postgres"
                 -   "dbt-redshift"
+                -   "dbt-snowflake"
                 -   "dbt-spark"
             branch:
                 description: "Choose the branch to build"

diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
@@ -13,7 +13,9 @@ on:
                 -   "dbt-athena"
                 -   "dbt-athena-community"
                 -   "dbt-bigquery"
+                -   "dbt-postgres"
                 -   "dbt-redshift"
+                -   "dbt-snowflake"
                 -   "dbt-spark"
             deploy-to:
                 description: "Choose whether to publish to test or prod"
@@ -31,6 +33,10 @@ on:
                 description: "Publish to PyPI"
                 type: boolean
                 default: false
+            skip-integration-tests:
+                description: "Skip Running Integration Tests"
+                type: boolean
+                default: false
 
 # don't publish to the same target in parallel
 concurrency:
@@ -45,7 +51,9 @@ jobs:
             branch: ${{ inputs.branch }}
 
     integration-tests:
-        if: ${{ inputs.package != 'dbt-adapters' && inputs.package != 'dbt-tests-adapter' }}
+        if: |
+            (inputs.package != 'dbt-adapters' && inputs.package != 'dbt-tests-adapter') &&
+            inputs.skip-integration-tests == false
         uses: ./.github/workflows/_integration-tests.yml
         with:
             package: ${{ inputs.package }}
@@ -56,9 +64,10 @@ jobs:
     generate-changelog:
         needs: [unit-tests, integration-tests]
         if: |
-          always() && 
+          always() &&
           (needs.integration-tests.result == 'skipped' || needs.integration-tests.result == 'success') &&
-          (needs.unit-tests.result == 'skipped' || needs.unit-tests.result == 'success')
+          (needs.unit-tests.result == 'skipped' || needs.unit-tests.result == 'success') &&
+          inputs.pypi-public == true
         uses: ./.github/workflows/_generate-changelog.yml
         with:
             package: ${{ inputs.package }}
@@ -67,8 +76,13 @@ jobs:
         secrets: inherit
 
     publish-internal:
-        if: ${{ inputs.pypi-internal == true && inputs.package != 'dbt-athena-community' }}
-        needs: generate-changelog
+        if: |
+          always() &&
+          (needs.integration-tests.result == 'skipped' || needs.integration-tests.result == 'success') &&
+          (needs.unit-tests.result == 'skipped' || needs.unit-tests.result == 'success') &&
+          inputs.pypi-internal == true &&
+          inputs.package != 'dbt-athena-community'
+        needs: [unit-tests, integration-tests]
         uses: ./.github/workflows/_publish-internal.yml
         with:
             package: ${{ inputs.package }}
@@ -83,7 +97,11 @@ jobs:
             package: ${{ inputs.package }}
 
     publish-pypi:
-        if: ${{ inputs.pypi-public == true }}
+        if: |
+          always() &&
+          (needs.integration-tests.result == 'skipped' || needs.integration-tests.result == 'success') &&
+          needs.generate-changelog.result == 'success' &&
+          inputs.pypi-public == true
         needs: [package, generate-changelog]
         runs-on: ${{ vars.DEFAULT_RUNNER }}
         environment:

diff --git a/.github/workflows/pull-request-checks.yml b/.github/workflows/pull-request-checks.yml
@@ -35,7 +35,9 @@ jobs:
                 -   "dbt-athena"
                 -   "dbt-athena-community"
                 -   "dbt-bigquery"
+                -   "dbt-postgres"
                 -   "dbt-redshift"
+                -   "dbt-snowflake"
                 -   "dbt-spark"
                 os: [ubuntu-22.04]
                 python-version: ["3.9", "3.10", "3.11", "3.12"]
@@ -56,7 +58,9 @@ jobs:
                 -   "dbt-athena"
                 -   "dbt-athena-community"
                 -   "dbt-bigquery"
+                -   "dbt-postgres"
                 -   "dbt-redshift"
+                -   "dbt-snowflake"
                 -   "dbt-spark"
                 os: [ ubuntu-22.04 ]
                 python-version: ["3.9", "3.10", "3.11", "3.12"]
@@ -76,7 +80,9 @@ jobs:
                 -   "dbt-athena"
                 -   "dbt-athena-community"
                 -   "dbt-bigquery"
+                -   "dbt-postgres"
                 -   "dbt-redshift"
+                -   "dbt-snowflake"
                 -   "dbt-spark"
                 os: [ubuntu-22.04]
                 python-version: ["3.9", "3.10", "3.11", "3.12"]

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -34,7 +34,7 @@ repos:
     rev: 7.1.1
     hooks:
     -   id: flake8
-        exclude: dbt/adapters/events/adapter_types_pb2.py|tests/functional/|dbt-spark/tests/|dbt-bigquery/tests/|dbt-redshift/tests
+        exclude: dbt/adapters/events/adapter_types_pb2.py|tests/functional/|dbt-spark/tests/|dbt-bigquery/tests/|dbt-redshift/tests|dbt-postgres/tests|dbt-snowflake/tests
         args:
         -   --max-line-length=99
         -   --select=E,F,W