Merge remote-tracking branch 'origin/monorepo/scheduled-testing' into…

… monorepo/scheduled-testing
dbt-labs · Jan 24, 2025 · 3acdd6b · 3acdd6b
2 parents b6ce602 + d0c44ba
commit 3acdd6b
Show file tree

Hide file tree

Showing 155 changed files with 9,211 additions and 4 deletions.
diff --git a/.changes/1.13.2.md b/.changes/1.13.2.md
@@ -0,0 +1 @@
+## dbt-adapters 1.13.2 - January 22, 2025
diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml
@@ -37,6 +37,8 @@ body:
             required: false
         -   label: dbt-bigquery
             required: false
+        -   label: dbt-redshift
+            required: false
         -   label: dbt-spark
             required: false
 -   type: textarea

diff --git a/.github/ISSUE_TEMPLATE/regression-report.yml b/.github/ISSUE_TEMPLATE/regression-report.yml
@@ -32,6 +32,8 @@ body:
             required: false
         -   label: dbt-bigquery
             required: false
+        -   label: dbt-redshift
+            required: false
         -   label: dbt-spark
             required: false
 -   type: textarea

diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -7,6 +7,7 @@ updates:
     -   "/dbt-athena"
     -   "/dbt-athena-community"
     -   "/dbt-bigquery"
+    -   "/dbt-redshift"
     -   "/dbt-spark"
     schedule:
         interval: "daily"
@@ -18,6 +19,7 @@ updates:
 -   package-ecosystem: "docker"
     directories:
     -   "/dbt-bigquery/docker"
+    -   "/dbt-redshift/docker"
     -   "/dbt-spark/docker"
     schedule:
         interval: "weekly"

diff --git a/.github/workflows/_generate-changelog.yml b/.github/workflows/_generate-changelog.yml
@@ -36,6 +36,7 @@ on:
                 -   "dbt-athena"
                 -   "dbt-athena-community"
                 -   "dbt-bigquery"
+                -   "dbt-redshift"
                 -   "dbt-spark"
             merge:
                 description: "Choose whether to merge the changelog branch"

diff --git a/.github/workflows/_integration-tests.yml b/.github/workflows/_integration-tests.yml
@@ -32,6 +32,7 @@ on:
                 -   "dbt-athena"
                 -   "dbt-athena-community"
                 -   "dbt-bigquery"
+                -   "dbt-redshift"
                 -   "dbt-spark"
             branch:
                 description: "Choose the branch to test"
@@ -127,6 +128,65 @@ jobs:
             if: ${{ inputs.python-version == '3.9' }}  # we only run this for one version to run in series
             working-directory: ./${{ inputs.package }}
 
+    integration-tests-redshift:
+        if: ${{ inputs.package == 'dbt-redshift' }}
+        runs-on: ${{ inputs.os }}
+        environment:
+            name: "dbt-redshift"
+        env:
+            AWS_USER_PROFILE: ${{ vars.AWS_USER_PROFILE }}
+            AWS_USER_ACCESS_KEY_ID: ${{ vars.AWS_USER_ACCESS_KEY_ID }}
+            AWS_USER_SECRET_ACCESS_KEY: ${{ secrets.AWS_USER_SECRET_ACCESS_KEY }}
+            AWS_SOURCE_PROFILE: ${{ vars.AWS_SOURCE_PROFILE }}
+            AWS_ROLE_PROFILE: ${{ vars.AWS_ROLE_PROFILE }}
+            AWS_ROLE_ACCESS_KEY_ID: ${{ vars.AWS_ROLE_ACCESS_KEY_ID }}
+            AWS_ROLE_SECRET_ACCESS_KEY: ${{ secrets.AWS_ROLE_SECRET_ACCESS_KEY }}
+            AWS_ROLE_ARN: ${{ secrets.AWS_ROLE_ARN }}
+            AWS_REGION: ${{ vars.AWS_REGION }}
+            REDSHIFT_TEST_DBNAME: ${{ vars.REDSHIFT_TEST_DBNAME }}
+            REDSHIFT_TEST_PASS: ${{ secrets.REDSHIFT_TEST_PASS }}
+            REDSHIFT_TEST_USER: ${{ vars.REDSHIFT_TEST_USER }}
+            REDSHIFT_TEST_PORT: ${{ vars.REDSHIFT_TEST_PORT }}
+            REDSHIFT_TEST_HOST: ${{ secrets.REDSHIFT_TEST_HOST }}
+            REDSHIFT_TEST_CLUSTER_ID: ${{ vars.REDSHIFT_TEST_CLUSTER_ID }}
+            REDSHIFT_TEST_REGION: ${{ vars.AWS_REGION }}
+            REDSHIFT_TEST_IAM_USER_PROFILE: ${{ vars.AWS_USER_PROFILE }}
+            REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID: ${{ vars.AWS_USER_ACCESS_KEY_ID }}
+            REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY: ${{ secrets.AWS_USER_SECRET_ACCESS_KEY }}
+            REDSHIFT_TEST_IAM_ROLE_PROFILE: ${{ vars.AWS_ROLE_PROFILE }}
+            DBT_TEST_USER_1: ${{ vars.DBT_TEST_USER_1 }}
+            DBT_TEST_USER_2: ${{ vars.DBT_TEST_USER_2 }}
+            DBT_TEST_USER_3: ${{ vars.DBT_TEST_USER_3 }}
+        steps:
+        -   uses: actions/checkout@v4
+            with:
+                ref: ${{ inputs.branch }}
+                repository: ${{ inputs.repository }}
+        -   uses: actions/setup-python@v5
+            with:
+                python-version: ${{ inputs.python-version }}
+        -   uses: pypa/hatch@install
+        -   name: Create AWS IAM profiles
+            run: |
+              aws configure --profile $AWS_USER_PROFILE set aws_access_key_id $AWS_USER_ACCESS_KEY_ID
+              aws configure --profile $AWS_USER_PROFILE set aws_secret_access_key $AWS_USER_SECRET_ACCESS_KEY
+              aws configure --profile $AWS_USER_PROFILE set region $AWS_REGION
+              aws configure --profile $AWS_USER_PROFILE set output json
+
+              aws configure --profile $AWS_SOURCE_PROFILE set aws_access_key_id $AWS_ROLE_ACCESS_KEY_ID
+              aws configure --profile $AWS_SOURCE_PROFILE set aws_secret_access_key $AWS_ROLE_SECRET_ACCESS_KEY
+              aws configure --profile $AWS_SOURCE_PROFILE set region $AWS_REGION
+              aws configure --profile $AWS_SOURCE_PROFILE set output json
+
+              aws configure --profile $AWS_ROLE_PROFILE set source_profile $AWS_SOURCE_PROFILE
+              aws configure --profile $AWS_ROLE_PROFILE set role_arn $AWS_ROLE_ARN
+              aws configure --profile $AWS_ROLE_PROFILE set region $AWS_REGION
+              aws configure --profile $AWS_ROLE_PROFILE set output json
+        -   run: hatch run integration-tests tests/functional -m "not flaky" --ddtrace
+            working-directory: ./${{ inputs.package }}
+        -   run: hatch run integration-tests tests/functional -m flaky -n1 --ddtrace
+            working-directory: ./${{ inputs.package }}
+
     integration-tests-spark:
         if: ${{ inputs.package == 'dbt-spark' }}
         runs-on: ${{ inputs.os }}

diff --git a/.github/workflows/_publish-internal.yml b/.github/workflows/_publish-internal.yml
@@ -24,6 +24,7 @@ on:
                 -   "dbt-adapters"
                 -   "dbt-athena"
                 -   "dbt-bigquery"
+                -   "dbt-redshift"
                 -   "dbt-spark"
             deploy-to:
                 description: "Choose whether to publish to test or prod"

diff --git a/.github/workflows/_publish-pypi.yml b/.github/workflows/_publish-pypi.yml
@@ -25,6 +25,7 @@ on:
                 -   "dbt-athena"
                 -   "dbt-athena-community"
                 -   "dbt-bigquery"
+                -   "dbt-redshift"
                 -   "dbt-spark"
             deploy-to:
                 description: "Choose whether to publish to test or prod"

diff --git a/.github/workflows/_unit-tests.yml b/.github/workflows/_unit-tests.yml
@@ -33,6 +33,7 @@ on:
                 -   "dbt-athena"
                 -   "dbt-athena-community"
                 -   "dbt-bigquery"
+                -   "dbt-redshift"
                 -   "dbt-spark"
             branch:
                 description: "Choose the branch to test"

diff --git a/.github/workflows/_verify-build.yml b/.github/workflows/_verify-build.yml
@@ -34,6 +34,7 @@ on:
                 -   "dbt-athena"
                 -   "dbt-athena-community"
                 -   "dbt-bigquery"
+                -   "dbt-redshift"
                 -   "dbt-spark"
             branch:
                 description: "Choose the branch to build"

diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
@@ -13,6 +13,7 @@ on:
                 -   "dbt-athena"
                 -   "dbt-athena-community"
                 -   "dbt-bigquery"
+                -   "dbt-redshift"
                 -   "dbt-spark"
             deploy-to:
                 description: "Choose whether to publish to test or prod"
@@ -54,12 +55,15 @@ jobs:
 
     generate-changelog:
         needs: [unit-tests, integration-tests]
+        if: |
+          always() && 
+          (needs.integration-tests.result == 'skipped' || needs.integration-tests.result == 'success') &&
+          (needs.unit-tests.result == 'skipped' || needs.unit-tests.result == 'success')
         uses: ./.github/workflows/_generate-changelog.yml
         with:
             package: ${{ inputs.package }}
             merge: ${{ inputs.deploy-to == 'prod' }}
             branch: ${{ inputs.branch }}
-            repository: ${{ github.repository }}
         secrets: inherit
 
     publish-internal:

diff --git a/.github/workflows/pull-request-checks.yml b/.github/workflows/pull-request-checks.yml
@@ -35,6 +35,7 @@ jobs:
                 -   "dbt-athena"
                 -   "dbt-athena-community"
                 -   "dbt-bigquery"
+                -   "dbt-redshift"
                 -   "dbt-spark"
                 os: [ubuntu-22.04, macos-14, windows-2022]
                 python-version: ["3.9", "3.12"]
@@ -55,6 +56,7 @@ jobs:
                 -   "dbt-athena"
                 -   "dbt-athena-community"
                 -   "dbt-bigquery"
+                -   "dbt-redshift"
                 -   "dbt-spark"
                 os: [ubuntu-22.04, macos-14, windows-2022]
                 python-version: ["3.9", "3.12"]
@@ -74,6 +76,7 @@ jobs:
                 -   "dbt-athena"
                 -   "dbt-athena-community"
                 -   "dbt-bigquery"
+                -   "dbt-redshift"
                 -   "dbt-spark"
                 os: [ubuntu-22.04]
                 python-version: ["3.9", "3.12"]

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -34,12 +34,13 @@ repos:
     rev: 7.1.1
     hooks:
     -   id: flake8
-        exclude: dbt/adapters/events/adapter_types_pb2.py|tests/functional/|dbt-spark/tests/|dbt-bigquery/tests/
+        exclude: dbt/adapters/events/adapter_types_pb2.py|tests/functional/|dbt-spark/tests/|dbt-bigquery/tests/|dbt-redshift/tests
         args:
         -   --max-line-length=99
         -   --select=E,F,W
         -   --ignore=E203,E501,E704,E741,W503,W504
         -   --per-file-ignores=*/__init__.py:F401,*/conftest.py:F401
+        additional_dependencies: [flaky]
 
 -   repo: https://github.com/pre-commit/mirrors-mypy
     rev: v1.11.2

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,6 +5,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html),
 and is generated by [Changie](https://github.com/miniscruff/changie).
 
+## dbt-adapters 1.13.2 - January 22, 2025
+
+
+
 ## dbt-adapters 1.13.1 - January 10, 2025
 
 ### Fixes
@@ -15,7 +19,6 @@ and is generated by [Changie](https://github.com/miniscruff/changie).
 ### Contributors
 - [@rongfengliang](https://github.com/rongfengliang) ([#187](https://github.com/dbt-labs/dbt-adapters/issues/187))
 
-
 ## dbt-adapters 1.13.0 - December 19, 2024
 
 ### Features

diff --git a/dbt-redshift/.changes/0.0.0.md b/dbt-redshift/.changes/0.0.0.md
@@ -0,0 +1,9 @@
+## Previous Releases
+For information on prior major and minor releases, see their changelogs:
+- [1.6](https://github.com/dbt-labs/dbt-redshift/blob/1.6.latest/CHANGELOG.md)
+- [1.5](https://github.com/dbt-labs/dbt-redshift/blob/1.5.latest/CHANGELOG.md)
+- [1.4](https://github.com/dbt-labs/dbt-redshift/blob/1.4.latest/CHANGELOG.md)
+- [1.3](https://github.com/dbt-labs/dbt-redshift/blob/1.3.latest/CHANGELOG.md)
+- [1.2](https://github.com/dbt-labs/dbt-redshift/blob/1.2.latest/CHANGELOG.md)
+- [1.1](https://github.com/dbt-labs/dbt-redshift/blob/1.1.latest/CHANGELOG.md)
+- [1.0](https://github.com/dbt-labs/dbt-redshift/blob/1.0.latest/CHANGELOG.md)
diff --git a/dbt-redshift/.changes/README.md b/dbt-redshift/.changes/README.md
@@ -0,0 +1,3 @@
+# CHANGELOG
+
+To view information about the changelog operation we suggest reading this [README](https://github.com/dbt-labs/dbt-redshift/blob/main/.changes/README.md) found in `dbt-redshift`.
diff --git a/dbt-redshift/.changes/header.tpl.md b/dbt-redshift/.changes/header.tpl.md
@@ -0,0 +1,6 @@
+# dbt-redshift Changelog
+
+- This file provides a full account of all changes to `dbt-redshift`.
+- Changes are listed under the (pre)release in which they first appear. Subsequent releases include changes from previous releases.
+- "Breaking changes" listed under a version may require action from end users or external maintainers when upgrading to that version.
+- Do not edit this file directly. This file is auto-generated using [changie](https://github.com/miniscruff/changie). For details on how to document a change, see [the contributing guide](https://github.com/dbt-labs/dbt-redshift/blob/main/CONTRIBUTING.md#adding-changelog-entry)
diff --git a/dbt-redshift/.changes/unreleased/unreleased/.gitkeep b/dbt-redshift/.changes/unreleased/unreleased/.gitkeep
diff --git a/dbt-redshift/.changes/unreleased/unreleased/Features-20241217-181340.yaml b/dbt-redshift/.changes/unreleased/unreleased/Features-20241217-181340.yaml
@@ -0,0 +1,6 @@
+kind: Features
+body: Add IdpTokenAuthPlugin authentication method.
+time: 2024-12-17T18:13:40.281494-08:00
+custom:
+    Author: versusfacit
+    Issue: "898"
diff --git a/dbt-redshift/.changes/unreleased/unreleased/Under the Hood-20241204-185729.yaml b/dbt-redshift/.changes/unreleased/unreleased/Under the Hood-20241204-185729.yaml
@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Add retry logic for retryable exceptions
+time: 2024-12-04T18:57:29.925299-08:00
+custom:
+    Author: versusfacit colin-rogers-dbt
+    Issue: "960"
diff --git a/dbt-redshift/.changes/unreleased/unreleased/Under the Hood-20241207-165918.yaml b/dbt-redshift/.changes/unreleased/unreleased/Under the Hood-20241207-165918.yaml
@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Move from setup.py to pyproject.toml and to hatch as a dev tool
+time: 2024-12-07T16:59:18.731819-05:00
+custom:
+  Author: mikealfare
+  Issue: "951"
diff --git a/dbt-redshift/.changes/unreleased/unreleased/Under the Hood-20241211-145132.yaml b/dbt-redshift/.changes/unreleased/unreleased/Under the Hood-20241211-145132.yaml
@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Refactor to use new batch context varaibles
+time: 2024-12-11T14:51:32.239224-06:00
+custom:
+  Author: QMalcolm
+  Issue: "966"
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,3 @@
		# CHANGELOG

		To view information about the changelog operation we suggest reading this [README](https://github.com/dbt-labs/dbt-redshift/blob/main/.changes/README.md) found in `dbt-redshift`.