Skip to content

Commit

Permalink
Merge pull request #610 from dbt-labs/monorepo/migrate-dbt-redshift
Browse files Browse the repository at this point in the history
ADAP-1131: Migrate dbt-redshift into dbt-adapters
  • Loading branch information
mikealfare authored Jan 22, 2025
2 parents 2081f07 + 0c40d81 commit 8a8aa48
Show file tree
Hide file tree
Showing 150 changed files with 9,130 additions and 1 deletion.
2 changes: 2 additions & 0 deletions .github/ISSUE_TEMPLATE/bug-report.yml
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,8 @@ body:
required: false
- label: dbt-bigquery
required: false
- label: dbt-redshift
required: false
- label: dbt-spark
required: false
- type: textarea
Expand Down
2 changes: 2 additions & 0 deletions .github/ISSUE_TEMPLATE/regression-report.yml
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@ body:
required: false
- label: dbt-bigquery
required: false
- label: dbt-redshift
required: false
- label: dbt-spark
required: false
- type: textarea
Expand Down
2 changes: 2 additions & 0 deletions .github/dependabot.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ updates:
- "/dbt-athena"
- "/dbt-athena-community"
- "/dbt-bigquery"
- "/dbt-redshift"
- "/dbt-spark"
schedule:
interval: "daily"
Expand All @@ -18,6 +19,7 @@ updates:
- package-ecosystem: "docker"
directories:
- "/dbt-bigquery/docker"
- "/dbt-redshift/docker"
- "/dbt-spark/docker"
schedule:
interval: "weekly"
Expand Down
1 change: 1 addition & 0 deletions .github/workflows/_generate-changelog.yml
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ on:
- "dbt-athena"
- "dbt-athena-community"
- "dbt-bigquery"
- "dbt-redshift"
- "dbt-spark"
merge:
description: "Choose whether to merge the changelog branch"
Expand Down
60 changes: 60 additions & 0 deletions .github/workflows/_integration-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ on:
- "dbt-athena"
- "dbt-athena-community"
- "dbt-bigquery"
- "dbt-redshift"
- "dbt-spark"
branch:
description: "Choose the branch to test"
Expand Down Expand Up @@ -127,6 +128,65 @@ jobs:
if: ${{ inputs.python-version == '3.9' }} # we only run this for one version to run in series
working-directory: ./${{ inputs.package }}

integration-tests-redshift:
if: ${{ inputs.package == 'dbt-redshift' }}
runs-on: ${{ inputs.os }}
environment:
name: "dbt-redshift"
env:
AWS_USER_PROFILE: ${{ vars.AWS_USER_PROFILE }}
AWS_USER_ACCESS_KEY_ID: ${{ vars.AWS_USER_ACCESS_KEY_ID }}
AWS_USER_SECRET_ACCESS_KEY: ${{ secrets.AWS_USER_SECRET_ACCESS_KEY }}
AWS_SOURCE_PROFILE: ${{ vars.AWS_SOURCE_PROFILE }}
AWS_ROLE_PROFILE: ${{ vars.AWS_ROLE_PROFILE }}
AWS_ROLE_ACCESS_KEY_ID: ${{ vars.AWS_ROLE_ACCESS_KEY_ID }}
AWS_ROLE_SECRET_ACCESS_KEY: ${{ secrets.AWS_ROLE_SECRET_ACCESS_KEY }}
AWS_ROLE_ARN: ${{ secrets.AWS_ROLE_ARN }}
AWS_REGION: ${{ vars.AWS_REGION }}
REDSHIFT_TEST_DBNAME: ${{ vars.REDSHIFT_TEST_DBNAME }}
REDSHIFT_TEST_PASS: ${{ secrets.REDSHIFT_TEST_PASS }}
REDSHIFT_TEST_USER: ${{ vars.REDSHIFT_TEST_USER }}
REDSHIFT_TEST_PORT: ${{ vars.REDSHIFT_TEST_PORT }}
REDSHIFT_TEST_HOST: ${{ secrets.REDSHIFT_TEST_HOST }}
REDSHIFT_TEST_CLUSTER_ID: ${{ vars.REDSHIFT_TEST_CLUSTER_ID }}
REDSHIFT_TEST_REGION: ${{ vars.AWS_REGION }}
REDSHIFT_TEST_IAM_USER_PROFILE: ${{ vars.AWS_USER_PROFILE }}
REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID: ${{ vars.AWS_USER_ACCESS_KEY_ID }}
REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY: ${{ secrets.AWS_USER_SECRET_ACCESS_KEY }}
REDSHIFT_TEST_IAM_ROLE_PROFILE: ${{ vars.AWS_ROLE_PROFILE }}
DBT_TEST_USER_1: ${{ vars.DBT_TEST_USER_1 }}
DBT_TEST_USER_2: ${{ vars.DBT_TEST_USER_2 }}
DBT_TEST_USER_3: ${{ vars.DBT_TEST_USER_3 }}
steps:
- uses: actions/checkout@v4
with:
ref: ${{ inputs.branch }}
repository: ${{ inputs.repository }}
- uses: actions/setup-python@v5
with:
python-version: ${{ inputs.python-version }}
- uses: pypa/hatch@install
- name: Create AWS IAM profiles
run: |
aws configure --profile $AWS_USER_PROFILE set aws_access_key_id $AWS_USER_ACCESS_KEY_ID
aws configure --profile $AWS_USER_PROFILE set aws_secret_access_key $AWS_USER_SECRET_ACCESS_KEY
aws configure --profile $AWS_USER_PROFILE set region $AWS_REGION
aws configure --profile $AWS_USER_PROFILE set output json
aws configure --profile $AWS_SOURCE_PROFILE set aws_access_key_id $AWS_ROLE_ACCESS_KEY_ID
aws configure --profile $AWS_SOURCE_PROFILE set aws_secret_access_key $AWS_ROLE_SECRET_ACCESS_KEY
aws configure --profile $AWS_SOURCE_PROFILE set region $AWS_REGION
aws configure --profile $AWS_SOURCE_PROFILE set output json
aws configure --profile $AWS_ROLE_PROFILE set source_profile $AWS_SOURCE_PROFILE
aws configure --profile $AWS_ROLE_PROFILE set role_arn $AWS_ROLE_ARN
aws configure --profile $AWS_ROLE_PROFILE set region $AWS_REGION
aws configure --profile $AWS_ROLE_PROFILE set output json
- run: hatch run integration-tests tests/functional -m "not flaky" --ddtrace
working-directory: ./${{ inputs.package }}
- run: hatch run integration-tests tests/functional -m flaky -n1 --ddtrace
working-directory: ./${{ inputs.package }}

integration-tests-spark:
if: ${{ inputs.package == 'dbt-spark' }}
runs-on: ${{ inputs.os }}
Expand Down
1 change: 1 addition & 0 deletions .github/workflows/_publish-internal.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ on:
- "dbt-adapters"
- "dbt-athena"
- "dbt-bigquery"
- "dbt-redshift"
- "dbt-spark"
deploy-to:
description: "Choose whether to publish to test or prod"
Expand Down
1 change: 1 addition & 0 deletions .github/workflows/_publish-pypi.yml
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ on:
- "dbt-athena"
- "dbt-athena-community"
- "dbt-bigquery"
- "dbt-redshift"
- "dbt-spark"
deploy-to:
description: "Choose whether to publish to test or prod"
Expand Down
1 change: 1 addition & 0 deletions .github/workflows/_unit-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ on:
- "dbt-athena"
- "dbt-athena-community"
- "dbt-bigquery"
- "dbt-redshift"
- "dbt-spark"
branch:
description: "Choose the branch to test"
Expand Down
1 change: 1 addition & 0 deletions .github/workflows/_verify-build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ on:
- "dbt-athena"
- "dbt-athena-community"
- "dbt-bigquery"
- "dbt-redshift"
- "dbt-spark"
branch:
description: "Choose the branch to build"
Expand Down
1 change: 1 addition & 0 deletions .github/workflows/publish.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ on:
- "dbt-athena"
- "dbt-athena-community"
- "dbt-bigquery"
- "dbt-redshift"
- "dbt-spark"
deploy-to:
description: "Choose whether to publish to test or prod"
Expand Down
3 changes: 3 additions & 0 deletions .github/workflows/pull-request-checks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ jobs:
- "dbt-athena"
- "dbt-athena-community"
- "dbt-bigquery"
- "dbt-redshift"
- "dbt-spark"
os: [ubuntu-22.04]
python-version: ["3.9", "3.10", "3.11", "3.12"]
Expand All @@ -55,6 +56,7 @@ jobs:
- "dbt-athena"
- "dbt-athena-community"
- "dbt-bigquery"
- "dbt-redshift"
- "dbt-spark"
os: [ ubuntu-22.04 ]
python-version: ["3.9", "3.10", "3.11", "3.12"]
Expand All @@ -74,6 +76,7 @@ jobs:
- "dbt-athena"
- "dbt-athena-community"
- "dbt-bigquery"
- "dbt-redshift"
- "dbt-spark"
os: [ubuntu-22.04]
python-version: ["3.9", "3.10", "3.11", "3.12"]
Expand Down
3 changes: 2 additions & 1 deletion .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -34,12 +34,13 @@ repos:
rev: 7.1.1
hooks:
- id: flake8
exclude: dbt/adapters/events/adapter_types_pb2.py|tests/functional/|dbt-spark/tests/|dbt-bigquery/tests/
exclude: dbt/adapters/events/adapter_types_pb2.py|tests/functional/|dbt-spark/tests/|dbt-bigquery/tests/|dbt-redshift/tests
args:
- --max-line-length=99
- --select=E,F,W
- --ignore=E203,E501,E704,E741,W503,W504
- --per-file-ignores=*/__init__.py:F401,*/conftest.py:F401
additional_dependencies: [flaky]

- repo: https://github.com/pre-commit/mirrors-mypy
rev: v1.11.2
Expand Down
9 changes: 9 additions & 0 deletions dbt-redshift/.changes/0.0.0.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
## Previous Releases
For information on prior major and minor releases, see their changelogs:
- [1.6](https://github.com/dbt-labs/dbt-redshift/blob/1.6.latest/CHANGELOG.md)
- [1.5](https://github.com/dbt-labs/dbt-redshift/blob/1.5.latest/CHANGELOG.md)
- [1.4](https://github.com/dbt-labs/dbt-redshift/blob/1.4.latest/CHANGELOG.md)
- [1.3](https://github.com/dbt-labs/dbt-redshift/blob/1.3.latest/CHANGELOG.md)
- [1.2](https://github.com/dbt-labs/dbt-redshift/blob/1.2.latest/CHANGELOG.md)
- [1.1](https://github.com/dbt-labs/dbt-redshift/blob/1.1.latest/CHANGELOG.md)
- [1.0](https://github.com/dbt-labs/dbt-redshift/blob/1.0.latest/CHANGELOG.md)
3 changes: 3 additions & 0 deletions dbt-redshift/.changes/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
# CHANGELOG

To view information about the changelog operation we suggest reading this [README](https://github.com/dbt-labs/dbt-redshift/blob/main/.changes/README.md) found in `dbt-redshift`.
6 changes: 6 additions & 0 deletions dbt-redshift/.changes/header.tpl.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
# dbt-redshift Changelog

- This file provides a full account of all changes to `dbt-redshift`.
- Changes are listed under the (pre)release in which they first appear. Subsequent releases include changes from previous releases.
- "Breaking changes" listed under a version may require action from end users or external maintainers when upgrading to that version.
- Do not edit this file directly. This file is auto-generated using [changie](https://github.com/miniscruff/changie). For details on how to document a change, see [the contributing guide](https://github.com/dbt-labs/dbt-redshift/blob/main/CONTRIBUTING.md#adding-changelog-entry)
Empty file.
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
kind: Features
body: Add IdpTokenAuthPlugin authentication method.
time: 2024-12-17T18:13:40.281494-08:00
custom:
Author: versusfacit
Issue: "898"
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
kind: Under the Hood
body: Add retry logic for retryable exceptions
time: 2024-12-04T18:57:29.925299-08:00
custom:
Author: versusfacit colin-rogers-dbt
Issue: "960"
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
kind: Under the Hood
body: Move from setup.py to pyproject.toml and to hatch as a dev tool
time: 2024-12-07T16:59:18.731819-05:00
custom:
Author: mikealfare
Issue: "951"
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
kind: Under the Hood
body: Refactor to use new batch context varaibles
time: 2024-12-11T14:51:32.239224-06:00
custom:
Author: QMalcolm
Issue: "966"
130 changes: 130 additions & 0 deletions dbt-redshift/.changie.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,130 @@
changesDir: .changes
unreleasedDir: unreleased
headerPath: header.tpl.md
versionHeaderPath: ""
changelogPath: CHANGELOG.md
versionExt: md
envPrefix: "CHANGIE_"
versionFormat: '## dbt-redshift {{.Version}} - {{.Time.Format "January 02, 2006"}}'
kindFormat: '### {{.Kind}}'
changeFormat: |-
{{- $IssueList := list }}
{{- $changes := splitList " " $.Custom.Issue }}
{{- range $issueNbr := $changes }}
{{- $changeLink := "[#nbr](https://github.com/dbt-labs/dbt-redshift/issues/nbr)" | replace "nbr" $issueNbr }}
{{- $IssueList = append $IssueList $changeLink }}
{{- end -}}
- {{.Body}} ({{ range $index, $element := $IssueList }}{{if $index}}, {{end}}{{$element}}{{end}})
kinds:
- label: Breaking Changes
- label: Features
- label: Fixes
- label: Under the Hood
- label: Dependencies
changeFormat: |-
{{- $PRList := list }}
{{- $changes := splitList " " $.Custom.PR }}
{{- range $pullrequest := $changes }}
{{- $changeLink := "[#nbr](https://github.com/dbt-labs/dbt-redshift/pull/nbr)" | replace "nbr" $pullrequest }}
{{- $PRList = append $PRList $changeLink }}
{{- end -}}
- {{.Body}} ({{ range $index, $element := $PRList }}{{if $index}}, {{end}}{{$element}}{{end}})
skipGlobalChoices: true
additionalChoices:
- key: Author
label: GitHub Username(s) (separated by a single space if multiple)
type: string
minLength: 3
- key: PR
label: GitHub Pull Request Number (separated by a single space if multiple)
type: string
minLength: 1
- label: Security
changeFormat: |-
{{- $PRList := list }}
{{- $changes := splitList " " $.Custom.PR }}
{{- range $pullrequest := $changes }}
{{- $changeLink := "[#nbr](https://github.com/dbt-labs/dbt-redshift/pull/nbr)" | replace "nbr" $pullrequest }}
{{- $PRList = append $PRList $changeLink }}
{{- end -}}
- {{.Body}} ({{ range $index, $element := $PRList }}{{if $index}}, {{end}}{{$element}}{{end}})
skipGlobalChoices: true
additionalChoices:
- key: Author
label: GitHub Username(s) (separated by a single space if multiple)
type: string
minLength: 3
- key: PR
label: GitHub Pull Request Number (separated by a single space if multiple)
type: string
minLength: 1

newlines:
afterChangelogHeader: 1
afterKind: 1
afterChangelogVersion: 1
beforeKind: 1
endOfVersion: 1

custom:
- key: Author
label: GitHub Username(s) (separated by a single space if multiple)
type: string
minLength: 3
- key: Issue
label: GitHub Issue Number (separated by a single space if multiple)
type: string
minLength: 1

footerFormat: |
{{- $contributorDict := dict }}
{{- /* ensure all names in this list are all lowercase for later matching purposes */}}
{{- $core_team := splitList " " .Env.CORE_TEAM }}
{{- /* ensure we always skip snyk and dependabot in addition to the core team */}}
{{- $maintainers := list "dependabot[bot]" "snyk-bot"}}
{{- range $team_member := $core_team }}
{{- $team_member_lower := lower $team_member }}
{{- $maintainers = append $maintainers $team_member_lower }}
{{- end }}
{{- range $change := .Changes }}
{{- $authorList := splitList " " $change.Custom.Author }}
{{- /* loop through all authors for a single changelog */}}
{{- range $author := $authorList }}
{{- $authorLower := lower $author }}
{{- /* we only want to include non-core team contributors */}}
{{- if not (has $authorLower $maintainers)}}
{{- $changeList := splitList " " $change.Custom.Author }}
{{- $IssueList := list }}
{{- $changeLink := $change.Kind }}
{{- if or (eq $change.Kind "Dependencies") (eq $change.Kind "Security") }}
{{- $changes := splitList " " $change.Custom.PR }}
{{- range $issueNbr := $changes }}
{{- $changeLink := "[#nbr](https://github.com/dbt-labs/dbt-redshift/pull/nbr)" | replace "nbr" $issueNbr }}
{{- $IssueList = append $IssueList $changeLink }}
{{- end -}}
{{- else }}
{{- $changes := splitList " " $change.Custom.Issue }}
{{- range $issueNbr := $changes }}
{{- $changeLink := "[#nbr](https://github.com/dbt-labs/dbt-redshift/issues/nbr)" | replace "nbr" $issueNbr }}
{{- $IssueList = append $IssueList $changeLink }}
{{- end -}}
{{- end }}
{{- /* check if this contributor has other changes associated with them already */}}
{{- if hasKey $contributorDict $author }}
{{- $contributionList := get $contributorDict $author }}
{{- $contributionList = concat $contributionList $IssueList }}
{{- $contributorDict := set $contributorDict $author $contributionList }}
{{- else }}
{{- $contributionList := $IssueList }}
{{- $contributorDict := set $contributorDict $author $contributionList }}
{{- end }}
{{- end}}
{{- end}}
{{- end }}
{{- /* no indentation here for formatting so the final markdown doesn't have unneeded indentations */}}
{{- if $contributorDict}}
### Contributors
{{- range $k,$v := $contributorDict }}
- [@{{$k}}](https://github.com/{{$k}}) ({{ range $index, $element := $v }}{{if $index}}, {{end}}{{$element}}{{end}})
{{- end }}
{{- end }}
16 changes: 16 additions & 0 deletions dbt-redshift/CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
# dbt-redshift Changelog

- This file provides a full account of all changes to `dbt-redshift`.
- Changes are listed under the (pre)release in which they first appear. Subsequent releases include changes from previous releases.
- "Breaking changes" listed under a version may require action from end users or external maintainers when upgrading to that version.
- Do not edit this file directly. This file is auto-generated using [changie](https://github.com/miniscruff/changie). For details on how to document a change, see [the contributing guide](https://github.com/dbt-labs/dbt-redshift/blob/main/CONTRIBUTING.md#adding-changelog-entry)

## Previous Releases
For information on prior major and minor releases, see their changelogs:
- [1.6](https://github.com/dbt-labs/dbt-redshift/blob/1.6.latest/CHANGELOG.md)
- [1.5](https://github.com/dbt-labs/dbt-redshift/blob/1.5.latest/CHANGELOG.md)
- [1.4](https://github.com/dbt-labs/dbt-redshift/blob/1.4.latest/CHANGELOG.md)
- [1.3](https://github.com/dbt-labs/dbt-redshift/blob/1.3.latest/CHANGELOG.md)
- [1.2](https://github.com/dbt-labs/dbt-redshift/blob/1.2.latest/CHANGELOG.md)
- [1.1](https://github.com/dbt-labs/dbt-redshift/blob/1.1.latest/CHANGELOG.md)
- [1.0](https://github.com/dbt-labs/dbt-redshift/blob/1.0.latest/CHANGELOG.md)
Loading

0 comments on commit 8a8aa48

Please sign in to comment.