diff --git a/.changes/0.1.0/Docs-20240109-131629.yaml b/.changes/0.1.0/Docs-20240109-131629.yaml deleted file mode 100644 index 22b2ad3f6..000000000 --- a/.changes/0.1.0/Docs-20240109-131629.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Docs -body: Configure `changie` -time: 2024-01-09T13:16:29.763021-05:00 -custom: - Author: mikealfare - Issue: 16 diff --git a/.changes/0.1.0/Docs-20240109-131736.yaml b/.changes/0.1.0/Docs-20240109-131736.yaml deleted file mode 100644 index 431869033..000000000 --- a/.changes/0.1.0/Docs-20240109-131736.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Docs -body: Setup ADR tracking framework -time: 2024-01-09T13:17:36.094147-05:00 -custom: - Author: mikealfare - Issue: "11" diff --git a/.changes/0.1.0/Docs-20240109-131858.yaml b/.changes/0.1.0/Docs-20240109-131858.yaml deleted file mode 100644 index decef9a7b..000000000 --- a/.changes/0.1.0/Docs-20240109-131858.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Docs -body: Create issue templates -time: 2024-01-09T13:18:58.11819-05:00 -custom: - Author: mikealfare - Issue: "12" diff --git a/.changes/0.1.0/Docs-20240109-131917.yaml b/.changes/0.1.0/Docs-20240109-131917.yaml deleted file mode 100644 index 3c5310601..000000000 --- a/.changes/0.1.0/Docs-20240109-131917.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Docs -body: Create PR template -time: 2024-01-09T13:19:17.749914-05:00 -custom: - Author: mikealfare - Issue: "13" diff --git a/.changes/0.1.0/Features-20240212-123544.yaml b/.changes/0.1.0/Features-20240212-123544.yaml deleted file mode 100644 index 239ad59f0..000000000 --- a/.changes/0.1.0/Features-20240212-123544.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Features -body: Update RelationConfig to capture all fields used by adapters -time: 2024-02-12T12:35:44.653555-08:00 -custom: - Author: colin-rogers-dbt - Issue: "30" diff --git a/.changes/0.1.0/Fixes-20240215-141545.yaml b/.changes/0.1.0/Fixes-20240215-141545.yaml deleted file mode 100644 index ced62f256..000000000 --- a/.changes/0.1.0/Fixes-20240215-141545.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Fixes -body: Ignore adapter-level support warnings for 'custom' constraints -time: 2024-02-15T14:15:45.764145+01:00 -custom: - Author: jtcohen6 - Issue: "90" diff --git a/.changes/0.1.0/Fixes-20240216-135420.yaml b/.changes/0.1.0/Fixes-20240216-135420.yaml deleted file mode 100644 index a04cd26b4..000000000 --- a/.changes/0.1.0/Fixes-20240216-135420.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Fixes -body: Make all adapter zone tests importable by removing "Test" prefix -time: 2024-02-16T13:54:20.411864-05:00 -custom: - Author: mikealfare - Issue: "93" diff --git a/.changes/0.1.0/Under the Hood-20240109-131958.yaml b/.changes/0.1.0/Under the Hood-20240109-131958.yaml deleted file mode 100644 index a062a2991..000000000 --- a/.changes/0.1.0/Under the Hood-20240109-131958.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Configure `dependabot` -time: 2024-01-09T13:19:58.060742-05:00 -custom: - Author: mikealfare - Issue: "14" diff --git a/.changes/0.1.0/Under the Hood-20240112-230236.yaml b/.changes/0.1.0/Under the Hood-20240112-230236.yaml deleted file mode 100644 index 1470ac6e9..000000000 --- a/.changes/0.1.0/Under the Hood-20240112-230236.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Implement unit testing in CI -time: 2024-01-12T23:02:36.630106-05:00 -custom: - Author: mikealfare - Issue: "10" diff --git a/.changes/0.1.0/Under the Hood-20240123-121220.yaml b/.changes/0.1.0/Under the Hood-20240123-121220.yaml deleted file mode 100644 index 8d01f2563..000000000 --- a/.changes/0.1.0/Under the Hood-20240123-121220.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Allow version to be specified in either __version__.py or __about__.py -time: 2024-01-23T12:12:20.529147-05:00 -custom: - Author: mikealfare - Issue: "44" diff --git a/.changes/0.1.0/Under the Hood-20240220-164223.yaml b/.changes/0.1.0/Under the Hood-20240220-164223.yaml deleted file mode 100644 index eefa441e6..000000000 --- a/.changes/0.1.0/Under the Hood-20240220-164223.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Remove __init__.py file from dbt.tests -time: 2024-02-20T16:42:23.706-05:00 -custom: - Author: gshank - Issue: "96" diff --git a/.changes/1.0.0.md b/.changes/1.0.0.md index b6cc44a9f..c46c81482 100644 --- a/.changes/1.0.0.md +++ b/.changes/1.0.0.md @@ -1,15 +1,32 @@ -## dbt-adapter 1.0.0 - April 01, 2024 +## dbt-adapters 1.0.0 - April 01, 2024 + +### Features + +* Update RelationConfig to capture all fields used by adapters ([#30](https://github.com/dbt-labs/dbt-adapters/issues/30)) ### Fixes -* Add field wrapper to BaseRelation members that were missing it. -* Add "description" and "meta" fields to RelationConfig protocol +* Add field wrapper to BaseRelation members that were missing it. ([#108](https://github.com/dbt-labs/dbt-adapters/issues/108)) +* Add "description" and "meta" fields to RelationConfig protocol ([#119](https://github.com/dbt-labs/dbt-adapters/issues/119)) +* Ignore adapter-level support warnings for 'custom' constraints ([#90](https://github.com/dbt-labs/dbt-adapters/issues/90)) +* Make all adapter zone tests importable by removing "Test" prefix ([#93](https://github.com/dbt-labs/dbt-adapters/issues/93)) + +### Docs + +* Configure `changie` ([#16](https://github.com/dbt-labs/dbt-adapters/issues/16)) +* Setup ADR tracking framework ([#11](https://github.com/dbt-labs/dbt-adapters/issues/11)) +* Create issue templates ([#12](https://github.com/dbt-labs/dbt-adapters/issues/12)) +* Create PR template ([#13](https://github.com/dbt-labs/dbt-adapters/issues/13)) ### Under the Hood -* Lazy load agate to improve dbt-core performance -* add BaseAdapater.MAX_SCHEMA_METADATA_RELATIONS +* Lazy load agate to improve dbt-core performance ([#125](https://github.com/dbt-labs/dbt-adapters/issues/125)) +* add BaseAdapater.MAX_SCHEMA_METADATA_RELATIONS ([#131](https://github.com/dbt-labs/dbt-adapters/issues/131)) +* Configure `dependabot` ([#14](https://github.com/dbt-labs/dbt-adapters/issues/14)) +* Implement unit testing in CI ([#22](https://github.com/dbt-labs/dbt-adapters/issues/22)) +* Allow version to be specified in either __version__.py or __about__.py ([#44](https://github.com/dbt-labs/dbt-adapters/issues/44)) +* Remove __init__.py file from dbt.tests ([#96](https://github.com/dbt-labs/dbt-adapters/issues/96)) ### Security -* Pin `black>=24.3` in `pyproject.toml` +* Pin `black>=24.3` in `pyproject.toml` ([#140](https://github.com/dbt-labs/dbt-adapters/issues/140)) diff --git a/.changes/1.1.0.md b/.changes/1.1.0.md index c43ef9aa9..224d8e85c 100644 --- a/.changes/1.1.0.md +++ b/.changes/1.1.0.md @@ -1,29 +1,29 @@ -## dbt-adapter 1.1.0 - May 01, 2024 +## dbt-adapters 1.1.0 - May 01, 2024 ### Features -* Debug log when `type_code` fails to convert to a `data_type` -* Introduce TableLastModifiedMetadataBatch and implement BaseAdapter.calculate_freshness_from_metadata_batch -* Support for sql fixtures in unit testing -* Cross-database `cast` macro -* Allow adapters to opt out of aliasing the subquery generated by render_limited -* subquery alias generated by render_limited now includes the relation name to mitigate duplicate aliasing +* Debug log when `type_code` fails to convert to a `data_type` ([#135](https://github.com/dbt-labs/dbt-adapters/issues/135)) +* Introduce TableLastModifiedMetadataBatch and implement BaseAdapter.calculate_freshness_from_metadata_batch ([#127](https://github.com/dbt-labs/dbt-adapters/issues/127)) +* Support for sql fixtures in unit testing ([#146](https://github.com/dbt-labs/dbt-adapters/issues/146)) +* Cross-database `cast` macro ([#173](https://github.com/dbt-labs/dbt-adapters/issues/173)) +* Allow adapters to opt out of aliasing the subquery generated by render_limited ([#179](https://github.com/dbt-labs/dbt-adapters/issues/179)) +* subquery alias generated by render_limited now includes the relation name to mitigate duplicate aliasing ([#179](https://github.com/dbt-labs/dbt-adapters/issues/179)) ### Fixes -* Fix adapter-specific cast handling for constraint enforcement +* Fix adapter-specific cast handling for constraint enforcement ([#165](https://github.com/dbt-labs/dbt-adapters/issues/165)) ### Docs -* Use `dbt-adapters` throughout the contributing guide +* Use `dbt-adapters` throughout the contributing guide ([#137](https://github.com/dbt-labs/dbt-adapters/issues/137)) ### Under the Hood -* Add the option to set the log level of the AdapterRegistered event -* Update dependabot config to cover GHA -* Validate that dbt-core and dbt-adapters remain de-coupled -* remove dbt_version from query comment test fixture +* Add the option to set the log level of the AdapterRegistered event ([#141](https://github.com/dbt-labs/dbt-adapters/issues/141)) +* Update dependabot config to cover GHA ([#161](https://github.com/dbt-labs/dbt-adapters/issues/161)) +* Validate that dbt-core and dbt-adapters remain de-coupled ([#174](https://github.com/dbt-labs/dbt-adapters/issues/174)) +* remove dbt_version from query comment test fixture ([#184](https://github.com/dbt-labs/dbt-adapters/issues/184)) ### Dependencies -* add support for py3.12 +* add support for py3.12 ([#185](https://github.com/dbt-labs/dbt-adapters/issues/185)) diff --git a/.changes/1.1.1.md b/.changes/1.1.1.md index 0a28c3ad0..9e590f948 100644 --- a/.changes/1.1.1.md +++ b/.changes/1.1.1.md @@ -1,5 +1,5 @@ -## dbt-adapter 1.1.1 - May 07, 2024 +## dbt-adapters 1.1.1 - May 07, 2024 ### Features -* Enable serialization contexts +* Enable serialization contexts ([#197](https://github.com/dbt-labs/dbt-adapters/issues/197)) diff --git a/.changes/1.2.1.md b/.changes/1.2.1.md index f838b7692..e554b90bf 100644 --- a/.changes/1.2.1.md +++ b/.changes/1.2.1.md @@ -1,10 +1,15 @@ -## dbt-adapter 1.2.1 - May 21, 2024 +## dbt-adapters 1.2.1 - May 21, 2024 ### Features -* Improvement of the compile error message in the get_fixture-sql.sql when the relation or the model not exist +* Improvement of the compile error message in the get_fixture-sql.sql when the relation or the model not exist ([#203](https://github.com/dbt-labs/dbt-adapters/issues/203)) +* Cross-database `date` macro ([#191](https://github.com/dbt-labs/dbt-adapters/issues/191)) + +### Fixes + +* Update Clone test to reflect core change removing `deferred` attribute from nodes ([#194](https://github.com/dbt-labs/dbt-adapters/issues/194)) ### Under the Hood -* Add query recording for adapters which use SQLConnectionManager -* Improve memory efficiency of process_results() +* Add query recording for adapters which use SQLConnectionManager ([#195](https://github.com/dbt-labs/dbt-adapters/issues/195)) +* Improve memory efficiency of process_results() ([#217](https://github.com/dbt-labs/dbt-adapters/issues/217)) diff --git a/.changes/1.3.0.md b/.changes/1.3.0.md new file mode 100644 index 000000000..6a23c3ba8 --- /dev/null +++ b/.changes/1.3.0.md @@ -0,0 +1,5 @@ +## dbt-adapters 1.3.0 - June 18, 2024 + +### Features + +* Add get_catalog_for_single_relation macro and capability to enable adapters to optimize catalog generation ([#231](https://github.com/dbt-labs/dbt-adapters/issues/231)) diff --git a/.changes/1.3.1.md b/.changes/1.3.1.md new file mode 100644 index 000000000..b8ec73740 --- /dev/null +++ b/.changes/1.3.1.md @@ -0,0 +1 @@ +## dbt-adapters 1.3.1 - June 20, 2024 diff --git a/.changes/1.3.2.md b/.changes/1.3.2.md new file mode 100644 index 000000000..6963a4c35 --- /dev/null +++ b/.changes/1.3.2.md @@ -0,0 +1,6 @@ +## dbt-adapters 1.3.2 - July 02, 2024 + +### Under the Hood + +* Fix query timer resolution ([#246](https://github.com/dbt-labs/dbt-adapters/issues/246)) +* Add optional release_connection parameter to connection_named method ([#247](https://github.com/dbt-labs/dbt-adapters/issues/247)) diff --git a/.changes/1.3.3.md b/.changes/1.3.3.md new file mode 100644 index 000000000..c62a05623 --- /dev/null +++ b/.changes/1.3.3.md @@ -0,0 +1,9 @@ +## dbt-adapters 1.3.3 - July 09, 2024 + +### Fixes + +* Fix scenario where using the `--empty` flag causes metadata queries to contain limit clauses ([#213](https://github.com/dbt-labs/dbt-adapters/issues/213)) + +### Under the Hood + +* --limit flag no longer subshells the query. This resolves the dbt Cloud experience issue where limit prevents ordering elements.. ([#207](https://github.com/dbt-labs/dbt-adapters/issues/207)) diff --git a/.changes/1.4.0.md b/.changes/1.4.0.md new file mode 100644 index 000000000..fc6279db2 --- /dev/null +++ b/.changes/1.4.0.md @@ -0,0 +1,13 @@ +## dbt-adapters 1.4.0 - July 30, 2024 + +### Features + +- render 'to' and 'to_columns' fields on foreign key constraints, and bump dbt-common lower bound to 1.6 ([#271](https://github.com/dbt-labs/dbt-adapters/issues/271)) + +### Fixes + +- Incremental table varchar column definition changed ([#276](https://github.com/dbt-labs/dbt-adapters/issues/276)) + +### Under the Hood + +- Rework record/replay to record at the database connection level. ([#244](https://github.com/dbt-labs/dbt-adapters/issues/244)) diff --git a/.changes/1.8.0.md b/.changes/1.8.0.md deleted file mode 100644 index bbe54825e..000000000 --- a/.changes/1.8.0.md +++ /dev/null @@ -1,9 +0,0 @@ -## dbt-adapter 1.8.0 - May 09, 2024 - -### Features - -* Cross-database `date` macro - -### Fixes - -* Update Clone test to reflect core change removing `deferred` attribute from nodes diff --git a/.changes/unreleased/Fixes-20240610-195300.yaml b/.changes/unreleased/Fixes-20240610-195300.yaml new file mode 100644 index 000000000..1f8cd5a59 --- /dev/null +++ b/.changes/unreleased/Fixes-20240610-195300.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Use model alias for the CTE identifier generated during ephemeral materialization +time: 2024-06-10T19:53:00.086488231Z +custom: + Author: jeancochrane + Issue: "5273" diff --git a/.changes/unreleased/Under the Hood-20240801-220551.yaml b/.changes/unreleased/Under the Hood-20240801-220551.yaml new file mode 100644 index 000000000..25b54a65f --- /dev/null +++ b/.changes/unreleased/Under the Hood-20240801-220551.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Updating changie.yaml to add contributors and PR links +time: 2024-08-01T22:05:51.327652-04:00 +custom: + Author: leahwicz + Issue: "219" diff --git a/.changie.yaml b/.changie.yaml index 9f78b81e1..8f1d86155 100644 --- a/.changie.yaml +++ b/.changie.yaml @@ -1,20 +1,65 @@ changesDir: .changes unreleasedDir: unreleased headerPath: header.tpl.md +versionHeaderPath: "" changelogPath: CHANGELOG.md versionExt: md -envPrefix: CHANGIE_ -versionFormat: '## dbt-adapter {{.Version}} - {{.Time.Format "January 02, 2006"}}' +envPrefix: "CHANGIE_" +versionFormat: '## dbt-adapters {{.Version}} - {{.Time.Format "January 02, 2006"}}' kindFormat: '### {{.Kind}}' -changeFormat: '* {{.Body}}' +changeFormat: |- + {{- $IssueList := list }} + {{- $changes := splitList " " $.Custom.Issue }} + {{- range $issueNbr := $changes }} + {{- $changeLink := "[#nbr](https://github.com/dbt-labs/dbt-adapters/issues/nbr)" | replace "nbr" $issueNbr }} + {{- $IssueList = append $IssueList $changeLink }} + {{- end -}} + - {{.Body}} ({{ range $index, $element := $IssueList }}{{if $index}}, {{end}}{{$element}}{{end}}) + kinds: - - label: Breaking Changes - - label: Features - - label: Fixes - - label: Docs - - label: Under the Hood - - label: Dependencies - - label: Security +- label: Breaking Changes +- label: Features +- label: Fixes +- label: Under the Hood +- label: Dependencies + changeFormat: |- + {{- $PRList := list }} + {{- $changes := splitList " " $.Custom.PR }} + {{- range $pullrequest := $changes }} + {{- $changeLink := "[#nbr](https://github.com/dbt-labs/dbt-adapters/pull/nbr)" | replace "nbr" $pullrequest }} + {{- $PRList = append $PRList $changeLink }} + {{- end -}} + - {{.Body}} ({{ range $index, $element := $PRList }}{{if $index}}, {{end}}{{$element}}{{end}}) + skipGlobalChoices: true + additionalChoices: + - key: Author + label: GitHub Username(s) (separated by a single space if multiple) + type: string + minLength: 3 + - key: PR + label: GitHub Pull Request Number (separated by a single space if multiple) + type: string + minLength: 1 +- label: Security + changeFormat: |- + {{- $PRList := list }} + {{- $changes := splitList " " $.Custom.PR }} + {{- range $pullrequest := $changes }} + {{- $changeLink := "[#nbr](https://github.com/dbt-labs/dbt-adapters/pull/nbr)" | replace "nbr" $pullrequest }} + {{- $PRList = append $PRList $changeLink }} + {{- end -}} + - {{.Body}} ({{ range $index, $element := $PRList }}{{if $index}}, {{end}}{{$element}}{{end}}) + skipGlobalChoices: true + additionalChoices: + - key: Author + label: GitHub Username(s) (separated by a single space if multiple) + type: string + minLength: 3 + - key: PR + label: GitHub Pull Request Number (separated by a single space if multiple) + type: string + minLength: 1 + newlines: afterChangelogHeader: 1 afterKind: 1 @@ -31,3 +76,57 @@ custom: label: GitHub Issue Number (separated by a single space if multiple) type: string minLength: 1 + + +footerFormat: | + {{- $contributorDict := dict }} + {{- /* ensure all names in this list are all lowercase for later matching purposes */}} + {{- $core_team := splitList " " .Env.CORE_TEAM }} + {{- /* ensure we always skip snyk and dependabot in addition to the core team */}} + {{- $maintainers := list "dependabot[bot]" "snyk-bot"}} + {{- range $team_member := $core_team }} + {{- $team_member_lower := lower $team_member }} + {{- $maintainers = append $maintainers $team_member_lower }} + {{- end }} + {{- range $change := .Changes }} + {{- $authorList := splitList " " $change.Custom.Author }} + {{- /* loop through all authors for a single changelog */}} + {{- range $author := $authorList }} + {{- $authorLower := lower $author }} + {{- /* we only want to include non-core team contributors */}} + {{- if not (has $authorLower $maintainers)}} + {{- $changeList := splitList " " $change.Custom.Author }} + {{- $IssueList := list }} + {{- $changeLink := $change.Kind }} + {{- if or (eq $change.Kind "Dependencies") (eq $change.Kind "Security") }} + {{- $changes := splitList " " $change.Custom.PR }} + {{- range $issueNbr := $changes }} + {{- $changeLink := "[#nbr](https://github.com/dbt-labs/dbt-adapters/pull/nbr)" | replace "nbr" $issueNbr }} + {{- $IssueList = append $IssueList $changeLink }} + {{- end -}} + {{- else }} + {{- $changes := splitList " " $change.Custom.Issue }} + {{- range $issueNbr := $changes }} + {{- $changeLink := "[#nbr](https://github.com/dbt-labs/dbt-adapters/issues/nbr)" | replace "nbr" $issueNbr }} + {{- $IssueList = append $IssueList $changeLink }} + {{- end -}} + {{- end }} + {{- /* check if this contributor has other changes associated with them already */}} + {{- if hasKey $contributorDict $author }} + {{- $contributionList := get $contributorDict $author }} + {{- $contributionList = concat $contributionList $IssueList }} + {{- $contributorDict := set $contributorDict $author $contributionList }} + {{- else }} + {{- $contributionList := $IssueList }} + {{- $contributorDict := set $contributorDict $author $contributionList }} + {{- end }} + {{- end}} + {{- end}} + {{- end }} + {{- /* no indentation here for formatting so the final markdown doesn't have unneeded indentations */}} + {{- if $contributorDict}} + ### Contributors + {{- range $k,$v := $contributorDict }} + - [@{{$k}}](https://github.com/{{$k}}) ({{ range $index, $element := $v }}{{if $index}}, {{end}}{{$element}}{{end}}) + {{- end }} + {{- end }} diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml index 3bd90bf71..27cb521e8 100644 --- a/.github/ISSUE_TEMPLATE/bug-report.yml +++ b/.github/ISSUE_TEMPLATE/bug-report.yml @@ -1,5 +1,5 @@ name: 🐞 Bug -description: Report a bug or an issue you've found with dbt-adapter +description: Report a bug or an issue you've found with dbt-adapters title: "[Bug] " labels: ["bug", "triage"] body: @@ -62,11 +62,11 @@ body: examples: - **OS**: Ubuntu 20.04 - **Python**: 3.11.6 (`python3 --version`) - - **dbt-adapter**: 1.0.0 + - **dbt-adapters**: 1.0.0 value: | - OS: - Python: - - dbt-adapter: + - dbt-adapters: render: markdown validations: required: false diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 2e23e0fd6..a89889afa 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -7,5 +7,5 @@ contact_links: url: mailto:support@getdbt.com about: Are you using dbt Cloud? Contact our support team for help! - name: Participate in Discussions - url: https://github.com/dbt-labs/dbt-adapter/discussions - about: Do you have a Big Idea for dbt-adapter? Read open discussions, or start a new one + url: https://github.com/dbt-labs/dbt-adapters/discussions + about: Do you have a Big Idea for dbt-adapters? Read open discussions, or start a new one diff --git a/.github/ISSUE_TEMPLATE/feature-request.yml b/.github/ISSUE_TEMPLATE/feature-request.yml index 25b28aae1..22960c2d6 100644 --- a/.github/ISSUE_TEMPLATE/feature-request.yml +++ b/.github/ISSUE_TEMPLATE/feature-request.yml @@ -1,5 +1,5 @@ name: ✨ Feature -description: Propose a straightforward extension of dbt-adapter functionality +description: Propose a straightforward extension of dbt-adapters functionality title: "[Feature] <title>" labels: ["enhancement", "triage"] body: @@ -14,15 +14,15 @@ body: We want to make sure that features are distinct and discoverable, so that other members of the community can find them and offer their thoughts. - Issues are the right place to request straightforward extensions of existing dbt-adapter functionality. - For "big ideas" about future capabilities of dbt-adapter, we ask that you open a - [discussion](https://github.com/dbt-labs/dbt-adapter/discussions/new?category=ideas) in the "Ideas" category instead. + Issues are the right place to request straightforward extensions of existing dbt-adapters functionality. + For "big ideas" about future capabilities of dbt-adapters, we ask that you open a + [discussion](https://github.com/dbt-labs/dbt-adapters/discussions/new?category=ideas) in the "Ideas" category instead. options: - label: I have read the [expectations for open source contributors](https://docs.getdbt.com/docs/contributing/oss-expectations) required: true - label: I have searched the existing issues, and I could not find an existing issue for this feature required: true - - label: I am requesting a straightforward extension of existing dbt-adapter functionality, rather than a Big Idea better suited to a discussion + - label: I am requesting a straightforward extension of existing dbt-adapters functionality, rather than a Big Idea better suited to a discussion required: true - type: textarea attributes: diff --git a/.github/ISSUE_TEMPLATE/regression-report.yml b/.github/ISSUE_TEMPLATE/regression-report.yml index 017755077..6831ede2f 100644 --- a/.github/ISSUE_TEMPLATE/regression-report.yml +++ b/.github/ISSUE_TEMPLATE/regression-report.yml @@ -1,5 +1,5 @@ name: ☣️ Regression -description: Report a regression you've observed in a newer version of dbt-adapter +description: Report a regression you've observed in a newer version of dbt-adapters title: "[Regression] <title>" labels: ["regression", "triage"] body: @@ -57,13 +57,13 @@ body: examples: - **OS**: Ubuntu 20.04 - **Python**: 3.11.6 (`python3 --version`) - - **dbt-adapter (working version)**: 1.1.0 - - **dbt-adapter (regression version)**: 1.2.0 + - **dbt-adapters (working version)**: 1.1.0 + - **dbt-adapters (regression version)**: 1.2.0 value: | - OS: - Python: - - dbt-adapter (working version): - - dbt-adapter (regression version): + - dbt-adapters (working version): + - dbt-adapters (regression version): render: markdown validations: required: true diff --git a/.github/actions/build-hatch/action.yml b/.github/actions/build-hatch/action.yml index fe9825d46..6d81339a6 100644 --- a/.github/actions/build-hatch/action.yml +++ b/.github/actions/build-hatch/action.yml @@ -13,7 +13,7 @@ inputs: default: "./" archive-name: description: Where to upload the artifacts - required: true + default: "" runs: using: composite @@ -30,7 +30,8 @@ runs: working-directory: ${{ inputs.working-dir }} - name: Upload artifacts - uses: actions/upload-artifact@v3 + if: ${{ inputs.archive-name != '' }} + uses: actions/upload-artifact@v4 with: name: ${{ inputs.archive-name }} path: ${{ inputs.working-dir }}dist/ diff --git a/.github/actions/publish-pypi/action.yml b/.github/actions/publish-pypi/action.yml index deffc6e36..25bc3a8d1 100644 --- a/.github/actions/publish-pypi/action.yml +++ b/.github/actions/publish-pypi/action.yml @@ -14,7 +14,7 @@ runs: steps: - name: Download artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: ${{ inputs.archive-name }} path: dist/ diff --git a/.github/actions/publish-results/action.yml b/.github/actions/publish-results/action.yml index d863d6598..0d5cb7e65 100644 --- a/.github/actions/publish-results/action.yml +++ b/.github/actions/publish-results/action.yml @@ -19,7 +19,7 @@ runs: run: echo "ts=$(date +'%Y-%m-%dT%H-%M-%S')" >> $GITHUB_OUTPUT #no colons allowed for artifacts shell: bash - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: name: ${{ inputs.file-name }}_python-${{ inputs.python-version }}_${{ steps.timestamp.outputs.ts }}.csv path: ${{ inputs.source-file }} diff --git a/.github/actions/setup-hatch/action.yml b/.github/actions/setup-hatch/action.yml index 6b15cdbf3..6bf8ea109 100644 --- a/.github/actions/setup-hatch/action.yml +++ b/.github/actions/setup-hatch/action.yml @@ -13,7 +13,7 @@ runs: using: composite steps: - name: Set up Python ${{ inputs.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ inputs.python-version }} diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 02f010c76..907926a30 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -5,13 +5,25 @@ updates: schedule: interval: "daily" rebase-strategy: "disabled" + ignore: + - dependency-name: "*" + update-types: + - version-update:semver-patch - package-ecosystem: "pip" directory: "/dbt-tests-adapter" schedule: interval: "daily" rebase-strategy: "disabled" + ignore: + - dependency-name: "*" + update-types: + - version-update:semver-patch - package-ecosystem: "github-actions" directory: "/" schedule: interval: "weekly" rebase-strategy: "disabled" + ignore: + - dependency-name: "*" + update-types: + - version-update:semver-patch diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 4fc2fcf8e..3879b6538 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -29,7 +29,7 @@ resolves # ### Checklist -- [ ] I have read [the contributing guide](https://github.com/dbt-labs/dbt-adapter/blob/main/CONTRIBUTING.md) and understand what's expected of me +- [ ] I have read [the contributing guide](https://github.com/dbt-labs/dbt-adapters/blob/main/CONTRIBUTING.md) and understand what's expected of me - [ ] I have run this code in development, and it appears to resolve the stated issue - [ ] This PR includes tests, or tests are not required/relevant for this PR - [ ] This PR has no interface changes (e.g. macros, cli, logs, json artifacts, config files, adapter interface, etc.) or this PR has already received feedback and approval from Product or DX diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 33d94ff48..00afd7040 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -20,11 +20,6 @@ on: types: [checks_requested] workflow_dispatch: workflow_call: - inputs: - changelog_path: - description: "Path to changelog file" - required: true - type: string permissions: read-all @@ -51,35 +46,9 @@ jobs: uses: ./.github/actions/setup-hatch - name: Build `dbt-adapters` - if: ${{ inputs.package == 'dbt-adapters' }} uses: ./.github/actions/build-hatch - name: Build `dbt-tests-adapter` - if: ${{ inputs.package == 'dbt-tests-adapter' }} uses: ./.github/actions/build-hatch with: working-dir: "./dbt-tests-adapter/" - - - name: Setup `hatch` - uses: ./.github/actions/setup-hatch - - - name: Build `dbt-adapters` - if: ${{ inputs.package == 'dbt-adapters' }} - uses: ./.github/actions/build-hatch - - - name: Build `dbt-tests-adapter` - if: ${{ inputs.package == 'dbt-tests-adapter' }} - uses: ./.github/actions/build-hatch - with: - working-dir: "./dbt-tests-adapter/" - - # this step is only needed for the release process - - name: "Upload Build Artifact" - if: ${{ github.event_name == 'workflow_call' }} - uses: actions/upload-artifact@v3 - with: - name: ${{ steps.version.outputs.version_number }} - path: | - ${{ inputs.changelog_path }} - ./dist/ - retention-days: 3 diff --git a/.github/workflows/changelog-existence.yml b/.github/workflows/changelog-existence.yml index d778f5655..8732177f2 100644 --- a/.github/workflows/changelog-existence.yml +++ b/.github/workflows/changelog-existence.yml @@ -19,9 +19,6 @@ name: Check Changelog Entry on: pull_request_target: types: [opened, reopened, labeled, unlabeled, synchronize] - paths-ignore: ['.changes/**', '.github/**', 'tests/**', 'third-party-stubs/**', '**.md', '**.yml'] - - workflow_dispatch: defaults: run: diff --git a/.github/workflows/docs-issue.yml b/.github/workflows/docs-issue.yml new file mode 100644 index 000000000..f49cf517c --- /dev/null +++ b/.github/workflows/docs-issue.yml @@ -0,0 +1,41 @@ +# **what?** +# Open an issue in docs.getdbt.com when an issue is labeled `user docs` and closed as completed + +# **why?** +# To reduce barriers for keeping docs up to date + +# **when?** +# When an issue is labeled `user docs` and is closed as completed. Can be labeled before or after the issue is closed. + + +name: Open issues in docs.getdbt.com repo when an issue is labeled +run-name: "Open an issue in docs.getdbt.com for issue #${{ github.event.issue.number }}" + +on: + issues: + types: [labeled, closed] + +defaults: + run: + shell: bash + +permissions: + issues: write # comments on issues + +jobs: + open_issues: + # we only want to run this when the issue is closed as completed and the label `user docs` has been assigned. + # If this logic does not exist in this workflow, it runs the + # risk of duplicaton of issues being created due to merge and label both triggering this workflow to run and neither having + # generating the comment before the other runs. This lives here instead of the shared workflow because this is where we + # decide if it should run or not. + if: | + (github.event.issue.state == 'closed' && github.event.issue.state_reason == 'completed') && ( + (github.event.action == 'closed' && contains(github.event.issue.labels.*.name, 'user docs')) || + (github.event.action == 'labeled' && github.event.label.name == 'user docs')) + uses: dbt-labs/actions/.github/workflows/open-issue-in-repo.yml@main + with: + issue_repository: "dbt-labs/docs.getdbt.com" + issue_title: "Docs Changes Needed from ${{ github.event.repository.name }} Issue #${{ github.event.issue.number }}" + issue_body: "At a minimum, update body to include a link to the page on docs.getdbt.com requiring updates and what part(s) of the page you would like to see updated." + secrets: inherit diff --git a/.github/workflows/github-release.yml b/.github/workflows/github-release.yml index 1c2f41b54..ad0cc2d82 100644 --- a/.github/workflows/github-release.yml +++ b/.github/workflows/github-release.yml @@ -208,7 +208,7 @@ jobs: ref: ${{ inputs.sha }} - name: "Download Artifact ${{ inputs.archive_name }}" - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: ${{ inputs.archive_name }} path: dist/ diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1135adb84..828350ddc 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -151,7 +151,7 @@ jobs: github-release: name: "GitHub Release" - # ToDo: update GH release to handle adding dbt-tests-adapter and dbt-adapter assets to the same release + # ToDo: update GH release to handle adding dbt-tests-adapter and dbt-adapters assets to the same release if: ${{ !failure() && !cancelled() && inputs.package == 'dbt-adapters' }} needs: [release-inputs, build-and-test, bump-version-generate-changelog] uses: dbt-labs/dbt-adapters/.github/workflows/github-release.yml@main diff --git a/.github/workflows/release_prep_hatch.yml b/.github/workflows/release_prep_hatch.yml index b043e19e9..a61057866 100644 --- a/.github/workflows/release_prep_hatch.yml +++ b/.github/workflows/release_prep_hatch.yml @@ -34,7 +34,7 @@ # name: Version Bump and Changelog Generation -run-name: Bump ${{ inputs.package }}==${{ inputs.version_number }} for release to ${{ inputs.deploy_to }} and generate changelog +run-name: Bump to ${{ inputs.version_number }} for release to ${{ inputs.deploy_to }} and generate changelog on: workflow_call: inputs: @@ -131,7 +131,7 @@ jobs: - name: "Audit Version And Parse Into Parts" id: semver - uses: dbt-labs/actions/parse-semver@v1.1.0 + uses: dbt-labs/actions/parse-semver@v1.1.1 with: version: ${{ inputs.version_number }} @@ -288,7 +288,7 @@ jobs: steps: - name: "Checkout ${{ github.repository }} Branch ${{ needs.create-temp-branch.outputs.branch_name }}" - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ needs.create-temp-branch.outputs.branch_name }} - name: Setup `hatch` @@ -392,13 +392,13 @@ jobs: steps: - name: "Checkout ${{ github.repository }} Branch ${{ needs.create-temp-branch.outputs.branch_name }}" - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ needs.create-temp-branch.outputs.branch_name }} - name: "Setup `hatch`" uses: ./.github/actions/setup-hatch - name: "Run Unit Tests" - run: hatch run unit-tests:all + run: hatch run unit-tests run-integration-tests: runs-on: ubuntu-20.04 @@ -407,7 +407,7 @@ jobs: steps: - name: "Checkout ${{ github.repository }} Branch ${{ needs.create-temp-branch.outputs.branch_name }}" - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ needs.create-temp-branch.outputs.branch_name }} @@ -447,7 +447,7 @@ jobs: python-version: ${{ env.PYTHON_TARGET_VERSION }} - name: Run tests - run: hatch run integration-tests:all + run: hatch run integration-tests merge-changes-into-target-branch: runs-on: ubuntu-latest @@ -467,7 +467,7 @@ jobs: echo needs.audit-changelog.outputs.exists: ${{ needs.audit-changelog.outputs.exists }} echo needs.audit-version-in-code.outputs.up_to_date: ${{ needs.audit-version-in-code.outputs.up_to_date }} - name: "Checkout Repo ${{ github.repository }}" - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: "Merge Changes Into ${{ inputs.target_branch }}" uses: everlytic/branch-merge@1.1.5 @@ -524,7 +524,7 @@ jobs: message="The ${{ steps.resolve_branch.outputs.target_branch }} branch will be used for release" echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - name: "Checkout Resolved Branch - ${{ steps.resolve_branch.outputs.target_branch }}" - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ steps.resolve_branch.outputs.target_branch }} diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 7d7206552..b61c83d71 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -37,7 +37,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Run unit tests - run: hatch run unit-tests:all + run: hatch run unit-tests shell: bash - name: Publish results diff --git a/.gitignore b/.gitignore index a14d6d0db..29c470c51 100644 --- a/.gitignore +++ b/.gitignore @@ -154,3 +154,13 @@ cython_debug/ # PyCharm .idea/ + +# MacOS +.DS_Store + +# VSCode +.vscode/ +.venv/ + +# Vim +*.swp diff --git a/CHANGELOG.md b/CHANGELOG.md index aaa477cf7..4146e95eb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,75 +5,126 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html), and is generated by [Changie](https://github.com/miniscruff/changie). -## dbt-adapter 1.8.0 - May 09, 2024 +## dbt-adapters 1.4.0 - July 30, 2024 ### Features -* Cross-database `date` macro +- render 'to' and 'to_columns' fields on foreign key constraints, and bump dbt-common lower bound to 1.6 ([#271](https://github.com/dbt-labs/dbt-adapters/issues/271)) ### Fixes -* Update Clone test to reflect core change removing `deferred` attribute from nodes +- Incremental table varchar column definition changed ([#276](https://github.com/dbt-labs/dbt-adapters/issues/276)) -## dbt-adapter 1.2.1 - May 21, 2024 +### Under the Hood + +- Rework record/replay to record at the database connection level. ([#244](https://github.com/dbt-labs/dbt-adapters/issues/244)) + +## dbt-adapters 1.3.3 - July 09, 2024 + +### Fixes + +* Fix scenario where using the `--empty` flag causes metadata queries to contain limit clauses ([#213](https://github.com/dbt-labs/dbt-adapters/issues/213)) + +### Under the Hood + +* --limit flag no longer subshells the query. This resolves the dbt Cloud experience issue where limit prevents ordering elements.. ([#207](https://github.com/dbt-labs/dbt-adapters/issues/207)) + +## dbt-adapters 1.3.2 - July 02, 2024 + +### Under the Hood + +* Fix query timer resolution ([#246](https://github.com/dbt-labs/dbt-adapters/issues/246)) +* Add optional release_connection parameter to connection_named method ([#247](https://github.com/dbt-labs/dbt-adapters/issues/247)) + +## dbt-adapters 1.3.1 - June 20, 2024 + +## dbt-adapters 1.3.0 - June 18, 2024 ### Features -* Improvement of the compile error message in the get_fixture-sql.sql when the relation or the model not exist +* Add get_catalog_for_single_relation macro and capability to enable adapters to optimize catalog generation ([#231](https://github.com/dbt-labs/dbt-adapters/issues/231)) + +## dbt-adapters 1.2.1 - May 21, 2024 + +### Features + +* Improvement of the compile error message in the get_fixture-sql.sql when the relation or the model not exist ([#203](https://github.com/dbt-labs/dbt-adapters/issues/203)) +* Cross-database `date` macro ([#191](https://github.com/dbt-labs/dbt-adapters/issues/191)) + +### Fixes + +* Update Clone test to reflect core change removing `deferred` attribute from nodes ([#194](https://github.com/dbt-labs/dbt-adapters/issues/194)) ### Under the Hood -* Add query recording for adapters which use SQLConnectionManager -* Improve memory efficiency of process_results() +* Add query recording for adapters which use SQLConnectionManager ([#195](https://github.com/dbt-labs/dbt-adapters/issues/195)) +* Improve memory efficiency of process_results() ([#217](https://github.com/dbt-labs/dbt-adapters/issues/217)) -## dbt-adapter 1.1.1 - May 07, 2024 +## dbt-adapters 1.1.1 - May 07, 2024 ### Features -* Enable serialization contexts +* Enable serialization contexts ([#197](https://github.com/dbt-labs/dbt-adapters/issues/197)) -## dbt-adapter 1.1.0 - May 01, 2024 +## dbt-adapters 1.1.0 - May 01, 2024 ### Features -* Debug log when `type_code` fails to convert to a `data_type` -* Introduce TableLastModifiedMetadataBatch and implement BaseAdapter.calculate_freshness_from_metadata_batch -* Support for sql fixtures in unit testing -* Cross-database `cast` macro -* Allow adapters to opt out of aliasing the subquery generated by render_limited -* subquery alias generated by render_limited now includes the relation name to mitigate duplicate aliasing +* Debug log when `type_code` fails to convert to a `data_type` ([#135](https://github.com/dbt-labs/dbt-adapters/issues/135)) +* Introduce TableLastModifiedMetadataBatch and implement BaseAdapter.calculate_freshness_from_metadata_batch ([#127](https://github.com/dbt-labs/dbt-adapters/issues/127)) +* Support for sql fixtures in unit testing ([#146](https://github.com/dbt-labs/dbt-adapters/issues/146)) +* Cross-database `cast` macro ([#173](https://github.com/dbt-labs/dbt-adapters/issues/173)) +* Allow adapters to opt out of aliasing the subquery generated by render_limited ([#179](https://github.com/dbt-labs/dbt-adapters/issues/179)) +* subquery alias generated by render_limited now includes the relation name to mitigate duplicate aliasing ([#179](https://github.com/dbt-labs/dbt-adapters/issues/179)) ### Fixes -* Fix adapter-specific cast handling for constraint enforcement +* Fix adapter-specific cast handling for constraint enforcement ([#165](https://github.com/dbt-labs/dbt-adapters/issues/165)) ### Docs -* Use `dbt-adapters` throughout the contributing guide +* Use `dbt-adapters` throughout the contributing guide ([#137](https://github.com/dbt-labs/dbt-adapters/issues/137)) ### Under the Hood -* Add the option to set the log level of the AdapterRegistered event -* Update dependabot config to cover GHA -* Validate that dbt-core and dbt-adapters remain de-coupled -* remove dbt_version from query comment test fixture +* Add the option to set the log level of the AdapterRegistered event ([#141](https://github.com/dbt-labs/dbt-adapters/issues/141)) +* Update dependabot config to cover GHA ([#161](https://github.com/dbt-labs/dbt-adapters/issues/161)) +* Validate that dbt-core and dbt-adapters remain de-coupled ([#174](https://github.com/dbt-labs/dbt-adapters/issues/174)) +* remove dbt_version from query comment test fixture ([#184](https://github.com/dbt-labs/dbt-adapters/issues/184)) ### Dependencies -* add support for py3.12 +* add support for py3.12 ([#185](https://github.com/dbt-labs/dbt-adapters/issues/185)) + +## dbt-adapters 1.0.0 - April 01, 2024 + +### Features -## dbt-adapter 1.0.0 - April 01, 2024 +* Update RelationConfig to capture all fields used by adapters ([#30](https://github.com/dbt-labs/dbt-adapters/issues/30)) ### Fixes -* Add field wrapper to BaseRelation members that were missing it. -* Add "description" and "meta" fields to RelationConfig protocol +* Add field wrapper to BaseRelation members that were missing it. ([#108](https://github.com/dbt-labs/dbt-adapters/issues/108)) +* Add "description" and "meta" fields to RelationConfig protocol ([#119](https://github.com/dbt-labs/dbt-adapters/issues/119)) +* Ignore adapter-level support warnings for 'custom' constraints ([#90](https://github.com/dbt-labs/dbt-adapters/issues/90)) +* Make all adapter zone tests importable by removing "Test" prefix ([#93](https://github.com/dbt-labs/dbt-adapters/issues/93)) + +### Docs + +* Configure `changie` ([#16](https://github.com/dbt-labs/dbt-adapters/issues/16)) +* Setup ADR tracking framework ([#11](https://github.com/dbt-labs/dbt-adapters/issues/11)) +* Create issue templates ([#12](https://github.com/dbt-labs/dbt-adapters/issues/12)) +* Create PR template ([#13](https://github.com/dbt-labs/dbt-adapters/issues/13)) ### Under the Hood -* Lazy load agate to improve dbt-core performance -* add BaseAdapater.MAX_SCHEMA_METADATA_RELATIONS +* Lazy load agate to improve dbt-core performance ([#125](https://github.com/dbt-labs/dbt-adapters/issues/125)) +* add BaseAdapater.MAX_SCHEMA_METADATA_RELATIONS ([#131](https://github.com/dbt-labs/dbt-adapters/issues/131)) +* Configure `dependabot` ([#14](https://github.com/dbt-labs/dbt-adapters/issues/14)) +* Implement unit testing in CI ([#22](https://github.com/dbt-labs/dbt-adapters/issues/22)) +* Allow version to be specified in either __version__.py or __about__.py ([#44](https://github.com/dbt-labs/dbt-adapters/issues/44)) +* Remove __init__.py file from dbt.tests ([#96](https://github.com/dbt-labs/dbt-adapters/issues/96)) ### Security -* Pin `black>=24.3` in `pyproject.toml` +* Pin `black>=24.3` in `pyproject.toml` ([#140](https://github.com/dbt-labs/dbt-adapters/issues/140)) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c87ea23d0..1a6e92a29 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -17,13 +17,10 @@ This guide assumes users are developing on a Linux or MacOS system. The following utilities are needed or will be installed in this guide: - `pip` -- `virturalenv` +- `hatch` - `git` - `changie` -If local functional testing is required, then a database instance -and appropriate credentials are also required. - In addition to this guide, users are highly encouraged to read the `dbt-core` [CONTRIBUTING.md](https://github.com/dbt-labs/dbt-core/blob/main/CONTRIBUTING.md). Almost all information there is applicable here. @@ -66,25 +63,39 @@ Rather than forking `dbt-labs/dbt-adapters`, use `dbt-labs/dbt-adapters` directl ### Installation -1. Ensure the latest version of `pip` is installed: +1. Ensure the latest versions of `pip` and `hatch` are installed: ```shell - pip install --upgrade pip + pip install --user --upgrade pip hatch + ``` +2. This step is optional, but it's recommended. Configure `hatch` to create its virtual environments in the project. Add this block to your `hatch` `config.toml` file: + ```toml + # MacOS: ~/Library/Application Support/hatch/config.toml + [dirs.env] + virtual = ".hatch" ``` -2. Configure and activate a virtual environment using `virtualenv` as described in -[Setting up an environment](https://github.com/dbt-labs/dbt-core/blob/HEAD/CONTRIBUTING.md#setting-up-an-environment) -3. Install `dbt-adapters` and development dependencies in the virtual environment + This makes `hatch` create all virtual environments in the project root inside of the directory `/.hatch`, similar to `/.tox` for `tox`. + It also makes it easier to add this environment as a runner in common IDEs like VSCode and PyCharm. +3. Create a `hatch` environment with all of the development dependencies and activate it: ```shell - pip install -e .[dev] + hatch run setup + hatch shell + ``` +4. Run any commands within the virtual environment by prefixing the command with `hatch run`: + ```shell + hatch run <command> ``` -When `dbt-adapters` is installed this way, any changes made to the `dbt-adapters` source code -will be reflected in the virtual environment immediately. +## Testing +`dbt-adapters` contains [code quality checks](https://github.com/dbt-labs/dbt-adapters/tree/main/.pre-commit-config.yaml) and [unit tests](https://github.com/dbt-labs/dbt-adapters/tree/main/tests/unit). +While `dbt-tests-adapter` is also hosted in this repo, it requires a concrete adapter to run. -## Testing +### Code quality -`dbt-adapters` contains [unit](https://github.com/dbt-labs/dbt-adapters/tree/main/tests/unit) -and [functional](https://github.com/dbt-labs/dbt-adapters/tree/main/tests/functional) tests. +Code quality checks can run with a single command: +```shell +hatch run code-quality +``` ### Unit tests @@ -93,43 +104,38 @@ Unit tests can be run locally without setting up a database connection: ```shell # Note: replace $strings with valid names +# run all unit tests +hatch run unit-test + # run all unit tests in a module -python -m pytest tests/unit/$test_file_name.py +hatch run unit-test tests/unit/$test_file_name.py + # run a specific unit test -python -m pytest tests/unit/$test_file_name.py::$test_class_name::$test_method_name +hatch run unit-test tests/unit/$test_file_name.py::$test_class_name::$test_method_name ``` -### Functional tests - -Functional tests require a database to test against. There are two primary ways to run functional tests: +### Testing against a development branch -- Tests will run automatically against a dbt Labs owned database during PR checks -- Tests can be run locally by configuring a `test.env` file with appropriate `ENV` variables: - ```shell - cp test.env.example test.env - $EDITOR test.env - ``` +Some changes require a change in `dbt-common` and `dbt-adapters`. +In that case, the dependency on `dbt-common` must be updated to point to the development branch. For example: -> **_WARNING:_** The parameters in `test.env` must link to a valid database. -> `test.env` is git-ignored, but be _extra_ careful to never check in credentials -> or other sensitive information when developing. +```toml +[tool.hatch.envs.default] +dependencies = [ + "dbt-common @ git+https://github.com/dbt-labs/dbt-common.git@my-dev-branch", + ..., +] +``` -Functional tests can be run locally with a valid database connection configured in `test.env`: +This will install `dbt-common` as a snapshot. In other words, if `my-dev-branch` is updated on GitHub, those updates will not be reflected locally. +In order to pick up those updates, the `hatch` environment(s) will need to be rebuilt: ```shell -# Note: replace $strings with valid names - -# run all functional tests in a directory -python -m pytest tests/functional/$test_directory -# run all functional tests in a module -python -m pytest tests/functional/$test_dir_and_filename.py -# run all functional tests in a class -python -m pytest tests/functional/$test_dir_and_filename.py::$test_class_name -# run a specific functional test -python -m pytest tests/functional/$test_dir_and_filename.py::$test_class_name::$test__method_name +exit +hatch env prune +hatch shell ``` - ## Documentation ### User documentation @@ -163,7 +169,7 @@ Remember to commit and push the file that's created. ### Signing the CLA -> **_NOTE:_** All contributors to `dbt-adapter` must sign the +> **_NOTE:_** All contributors to `dbt-adapters` must sign the > [Contributor License Agreement](https://docs.getdbt.com/docs/contributor-license-agreements)(CLA). Maintainers will be unable to merge contributions until the contributor signs the CLA. diff --git a/dbt-tests-adapter/dbt/tests/__about__.py b/dbt-tests-adapter/dbt/tests/__about__.py index 6aaa73b80..1b022739e 100644 --- a/dbt-tests-adapter/dbt/tests/__about__.py +++ b/dbt-tests-adapter/dbt/tests/__about__.py @@ -1 +1 @@ -version = "1.8.0" +version = "1.9.2" diff --git a/dbt-tests-adapter/dbt/tests/adapter/basic/test_get_catalog_for_single_relation.py b/dbt-tests-adapter/dbt/tests/adapter/basic/test_get_catalog_for_single_relation.py new file mode 100644 index 000000000..78055cc59 --- /dev/null +++ b/dbt-tests-adapter/dbt/tests/adapter/basic/test_get_catalog_for_single_relation.py @@ -0,0 +1,87 @@ +import pytest + +from dbt.tests.util import run_dbt, get_connection + +models__my_table_model_sql = """ +select * from {{ ref('my_seed') }} +""" + + +models__my_view_model_sql = """ +{{ + config( + materialized='view', + ) +}} + +select * from {{ ref('my_seed') }} +""" + +seed__my_seed_csv = """id,first_name,email,ip_address,updated_at +1,Larry,lking0@miitbeian.gov.cn,69.135.206.194,2008-09-12 19:08:31 +""" + + +class BaseGetCatalogForSingleRelation: + @pytest.fixture(scope="class") + def project_config_update(self): + return {"name": "get_catalog_for_single_relation"} + + @pytest.fixture(scope="class") + def seeds(self): + return { + "my_seed.csv": seed__my_seed_csv, + } + + @pytest.fixture(scope="class") + def models(self): + return { + "my_view_model.sql": models__my_view_model_sql, + "my_table_model.sql": models__my_table_model_sql, + } + + @pytest.fixture(scope="class") + def expected_catalog_my_seed(self, project): + raise NotImplementedError( + "To use this test, please implement `get_catalog_for_single_relation`, inherited from `SQLAdapter`." + ) + + @pytest.fixture(scope="class") + def expected_catalog_my_model(self, project): + raise NotImplementedError( + "To use this test, please implement `get_catalog_for_single_relation`, inherited from `SQLAdapter`." + ) + + def get_relation_for_identifier(self, project, identifier): + return project.adapter.get_relation( + database=project.database, + schema=project.test_schema, + identifier=identifier, + ) + + def test_get_catalog_for_single_relation( + self, project, expected_catalog_my_seed, expected_catalog_my_view_model + ): + results = run_dbt(["seed"]) + assert len(results) == 1 + + my_seed_relation = self.get_relation_for_identifier(project, "my_seed") + + with get_connection(project.adapter): + actual_catalog_my_seed = project.adapter.get_catalog_for_single_relation( + my_seed_relation + ) + + assert actual_catalog_my_seed == expected_catalog_my_seed + + results = run_dbt(["run"]) + assert len(results) == 2 + + my_view_model_relation = self.get_relation_for_identifier(project, "my_view_model") + + with get_connection(project.adapter): + actual_catalog_my_view_model = project.adapter.get_catalog_for_single_relation( + my_view_model_relation + ) + + assert actual_catalog_my_view_model == expected_catalog_my_view_model diff --git a/dbt-tests-adapter/dbt/tests/adapter/constraints/fixtures.py b/dbt-tests-adapter/dbt/tests/adapter/constraints/fixtures.py index cfbd53796..2a4f089bc 100644 --- a/dbt-tests-adapter/dbt/tests/adapter/constraints/fixtures.py +++ b/dbt-tests-adapter/dbt/tests/adapter/constraints/fixtures.py @@ -363,7 +363,8 @@ - type: check expression: id >= 1 - type: foreign_key - expression: {schema}.foreign_key_model (id) + to: ref('foreign_key_model') + to_columns: ["id"] - type: unique data_tests: - unique diff --git a/dbt-tests-adapter/dbt/tests/adapter/dbt_show/test_dbt_show.py b/dbt-tests-adapter/dbt/tests/adapter/dbt_show/test_dbt_show.py index c00491aab..3f3d36c59 100644 --- a/dbt-tests-adapter/dbt/tests/adapter/dbt_show/test_dbt_show.py +++ b/dbt-tests-adapter/dbt/tests/adapter/dbt_show/test_dbt_show.py @@ -1,5 +1,6 @@ import pytest +from dbt_common.exceptions import DbtRuntimeError from dbt.tests.adapter.dbt_show import fixtures from dbt.tests.util import run_dbt @@ -47,9 +48,25 @@ def test_sql_header(self, project): run_dbt(["show", "--select", "sql_header", "--vars", "timezone: Asia/Kolkata"]) +class BaseShowDoesNotHandleDoubleLimit: + """see issue: https://github.com/dbt-labs/dbt-adapters/issues/207""" + + DATABASE_ERROR_MESSAGE = 'syntax error at or near "limit"' + + def test_double_limit_throws_syntax_error(self, project): + with pytest.raises(DbtRuntimeError) as e: + run_dbt(["show", "--limit", "1", "--inline", "select 1 limit 1"]) + + assert self.DATABASE_ERROR_MESSAGE in str(e) + + class TestPostgresShowSqlHeader(BaseShowSqlHeader): pass class TestPostgresShowLimit(BaseShowLimit): pass + + +class TestShowDoesNotHandleDoubleLimit(BaseShowDoesNotHandleDoubleLimit): + pass diff --git a/dbt-tests-adapter/dbt/tests/adapter/empty/_models.py b/dbt-tests-adapter/dbt/tests/adapter/empty/_models.py new file mode 100644 index 000000000..f5e684f7f --- /dev/null +++ b/dbt-tests-adapter/dbt/tests/adapter/empty/_models.py @@ -0,0 +1,111 @@ +model_input_sql = """ +select 1 as id +""" + +ephemeral_model_input_sql = """ +{{ config(materialized='ephemeral') }} +select 2 as id +""" + +raw_source_csv = """id +3 +""" + + +model_sql = """ +select * +from {{ ref('model_input') }} +union all +select * +from {{ ref('ephemeral_model_input') }} +union all +select * +from {{ source('seed_sources', 'raw_source') }} +""" + + +model_inline_sql = """ +select * from {{ source('seed_sources', 'raw_source') }} as raw_source +""" + +schema_sources_yml = """ +sources: + - name: seed_sources + schema: "{{ target.schema }}" + tables: + - name: raw_source +""" + + +SEED = """ +my_id,my_value +1,a +2,b +3,c +""".strip() + + +SCHEMA = """ +version: 2 + +seeds: + - name: my_seed + description: "This is my_seed" + columns: + - name: id + description: "This is my_seed.my_id" +""" + +CONTROL = """ +select * from {{ ref("my_seed") }} +""" + + +GET_COLUMNS_IN_RELATION = """ +{{ config(materialized="table") }} +{% set columns = adapter.get_columns_in_relation(ref("my_seed")) %} +select * from {{ ref("my_seed") }} +""" + + +ALTER_COLUMN_TYPE = """ +{{ config(materialized="table") }} +{{ alter_column_type(ref("my_seed"), "MY_VALUE", "varchar") }} +select * from {{ ref("my_seed") }} +""" + + +ALTER_RELATION_COMMENT = """ +{{ config( + materialized="table", + persist_docs={"relations": True}, +) }} +select * from {{ ref("my_seed") }} +""" + + +ALTER_COLUMN_COMMENT = """ +{{ config( + materialized="table", + persist_docs={"columns": True}, +) }} +select * from {{ ref("my_seed") }} +""" + + +ALTER_RELATION_ADD_REMOVE_COLUMNS = """ +{{ config(materialized="table") }} +{% set my_seed = adapter.Relation.create(this.database, this.schema, "my_seed", "table") %} +{% set my_column = api.Column("my_column", "varchar") %} +{% do alter_relation_add_remove_columns(my_seed, [my_column], none) %} +{% do alter_relation_add_remove_columns(my_seed, none, [my_column]) %} +select * from {{ ref("my_seed") }} +""" + + +TRUNCATE_RELATION = """ +{{ config(materialized="table") }} +{% set my_seed = adapter.Relation.create(this.database, this.schema, "my_seed", "table") %} +{{ truncate_relation(my_seed) }} +select * from {{ ref("my_seed") }} +""" diff --git a/dbt-tests-adapter/dbt/tests/adapter/empty/test_empty.py b/dbt-tests-adapter/dbt/tests/adapter/empty/test_empty.py index 2249d98d5..de15bd5b0 100644 --- a/dbt-tests-adapter/dbt/tests/adapter/empty/test_empty.py +++ b/dbt-tests-adapter/dbt/tests/adapter/empty/test_empty.py @@ -1,57 +1,23 @@ -import pytest - from dbt.tests.util import relation_from_name, run_dbt +import pytest - -model_input_sql = """ -select 1 as id -""" - -ephemeral_model_input_sql = """ -{{ config(materialized='ephemeral') }} -select 2 as id -""" - -raw_source_csv = """id -3 -""" - - -model_sql = """ -select * -from {{ ref('model_input') }} -union all -select * -from {{ ref('ephemeral_model_input') }} -union all -select * -from {{ source('seed_sources', 'raw_source') }} -""" - - -schema_sources_yml = """ -sources: - - name: seed_sources - schema: "{{ target.schema }}" - tables: - - name: raw_source -""" +from dbt.tests.adapter.empty import _models class BaseTestEmpty: @pytest.fixture(scope="class") def seeds(self): return { - "raw_source.csv": raw_source_csv, + "raw_source.csv": _models.raw_source_csv, } @pytest.fixture(scope="class") def models(self): return { - "model_input.sql": model_input_sql, - "ephemeral_model_input.sql": ephemeral_model_input_sql, - "model.sql": model_sql, - "sources.yml": schema_sources_yml, + "model_input.sql": _models.model_input_sql, + "ephemeral_model_input.sql": _models.ephemeral_model_input_sql, + "model.sql": _models.model_sql, + "sources.yml": _models.schema_sources_yml, } def assert_row_count(self, project, relation_name: str, expected_row_count: int): @@ -75,13 +41,9 @@ def test_run_with_empty(self, project): class BaseTestEmptyInlineSourceRef(BaseTestEmpty): @pytest.fixture(scope="class") def models(self): - model_sql = """ - select * from {{ source('seed_sources', 'raw_source') }} as raw_source - """ - return { - "model.sql": model_sql, - "sources.yml": schema_sources_yml, + "model.sql": _models.model_inline_sql, + "sources.yml": _models.schema_sources_yml, } def test_run_with_empty(self, project): @@ -92,4 +54,47 @@ def test_run_with_empty(self, project): class TestEmpty(BaseTestEmpty): + """ + Though we don't create these classes anymore, we need to keep this one in case an adapter wanted to import the test as-is to automatically run it. + We should consider adding a deprecation warning that suggests moving this into the concrete adapter and importing `BaseTestEmpty` instead. + """ + pass + + +class MetadataWithEmptyFlag: + @pytest.fixture(scope="class") + def seeds(self): + return {"my_seed.csv": _models.SEED} + + @pytest.fixture(scope="class") + def models(self): + return { + "schema.yml": _models.SCHEMA, + "control.sql": _models.CONTROL, + "get_columns_in_relation.sql": _models.GET_COLUMNS_IN_RELATION, + "alter_column_type.sql": _models.ALTER_COLUMN_TYPE, + "alter_relation_comment.sql": _models.ALTER_RELATION_COMMENT, + "alter_column_comment.sql": _models.ALTER_COLUMN_COMMENT, + "alter_relation_add_remove_columns.sql": _models.ALTER_RELATION_ADD_REMOVE_COLUMNS, + "truncate_relation.sql": _models.TRUNCATE_RELATION, + } + + @pytest.fixture(scope="class", autouse=True) + def setup(self, project): + run_dbt(["seed"]) + + @pytest.mark.parametrize( + "model", + [ + "control", + "get_columns_in_relation", + "alter_column_type", + "alter_relation_comment", + "alter_column_comment", + "alter_relation_add_remove_columns", + "truncate_relation", + ], + ) + def test_run(self, project, model): + run_dbt(["run", "--empty", "--select", model]) diff --git a/dbt-tests-adapter/pyproject.toml b/dbt-tests-adapter/pyproject.toml index 20342078a..73bce49e9 100644 --- a/dbt-tests-adapter/pyproject.toml +++ b/dbt-tests-adapter/pyproject.toml @@ -32,14 +32,6 @@ dependencies = [ "dbt-adapters", "pyyaml", ] - -[project.optional-dependencies] -build = [ - "wheel", - "twine", - "check-wheel-contents", -] - [project.urls] Homepage = "https://github.com/dbt-labs/dbt-adapters" Documentation = "https://docs.getdbt.com" @@ -62,7 +54,11 @@ include = ["dbt/tests", "dbt/__init__.py"] [tool.hatch.envs.build] detached = true -features = ["build"] +dependencies = [ + "wheel", + "twine", + "check-wheel-contents", +] [tool.hatch.envs.build.scripts] check-all = [ "- check-wheel", diff --git a/dbt/adapters/__about__.py b/dbt/adapters/__about__.py index eb1d9a0ff..d619c757e 100644 --- a/dbt/adapters/__about__.py +++ b/dbt/adapters/__about__.py @@ -1 +1 @@ -version = "1.2.1" +version = "1.4.0" diff --git a/dbt/adapters/base/column.py b/dbt/adapters/base/column.py index e2e6e1e0e..195684a47 100644 --- a/dbt/adapters/base/column.py +++ b/dbt/adapters/base/column.py @@ -123,9 +123,6 @@ def numeric_type(cls, dtype: str, precision: Any, scale: Any) -> str: else: return "{}({},{})".format(dtype, precision, scale) - def __repr__(self) -> str: - return "<Column {} ({})>".format(self.name, self.data_type) - @classmethod def from_description(cls, name: str, raw_data_type: str) -> "Column": match = re.match(r"([^(]+)(\([^)]+\))?", raw_data_type) diff --git a/dbt/adapters/base/impl.py b/dbt/adapters/base/impl.py index f58f8aba0..e0627c472 100644 --- a/dbt/adapters/base/impl.py +++ b/dbt/adapters/base/impl.py @@ -1,10 +1,10 @@ import abc +import time from concurrent.futures import as_completed, Future from contextlib import contextmanager from datetime import datetime from enum import Enum from multiprocessing.context import SpawnContext -import time from typing import ( Any, Callable, @@ -23,12 +23,15 @@ TYPE_CHECKING, ) +import pytz from dbt_common.clients.jinja import CallableMacroGenerator from dbt_common.contracts.constraints import ( ColumnLevelConstraint, ConstraintType, ModelLevelConstraint, ) +from dbt_common.contracts.metadata import CatalogTable +from dbt_common.events.functions import fire_event, warn_or_error from dbt_common.exceptions import ( DbtInternalError, DbtRuntimeError, @@ -38,14 +41,12 @@ NotImplementedError, UnexpectedNullError, ) -from dbt_common.events.functions import fire_event, warn_or_error from dbt_common.utils import ( AttrDict, cast_to_str, executor, filter_null_values, ) -import pytz from dbt.adapters.base.column import Column as BaseColumn from dbt.adapters.base.connections import ( @@ -222,6 +223,7 @@ class BaseAdapter(metaclass=AdapterMeta): - truncate_relation - rename_relation - get_columns_in_relation + - get_catalog_for_single_relation - get_column_schema_from_query - expand_column_types - list_relations_without_caching @@ -317,14 +319,18 @@ def nice_connection_name(self) -> str: return conn.name @contextmanager - def connection_named(self, name: str, query_header_context: Any = None) -> Iterator[None]: + def connection_named( + self, name: str, query_header_context: Any = None, should_release_connection=True + ) -> Iterator[None]: try: if self.connections.query_header is not None: self.connections.query_header.set(name, query_header_context) self.acquire_connection(name) yield finally: - self.release_connection() + if should_release_connection: + self.release_connection() + if self.connections.query_header is not None: self.connections.query_header.reset() @@ -627,6 +633,12 @@ def get_columns_in_relation(self, relation: BaseRelation) -> List[BaseColumn]: """Get a list of the columns in the given Relation.""" raise NotImplementedError("`get_columns_in_relation` is not implemented for this adapter!") + def get_catalog_for_single_relation(self, relation: BaseRelation) -> Optional[CatalogTable]: + """Get catalog information including table-level and column-level metadata for a single relation.""" + raise NotImplementedError( + "`get_catalog_for_single_relation` is not implemented for this adapter!" + ) + @available.deprecated("get_columns_in_relation", lambda *a, **k: []) def get_columns_in_table(self, schema: str, identifier: str) -> List[BaseColumn]: """DEPRECATED: Get a list of the columns in the given table.""" @@ -1590,8 +1602,13 @@ def render_column_constraint(cls, constraint: ColumnLevelConstraint) -> Optional rendered_column_constraint = f"unique {constraint_expression}" elif constraint.type == ConstraintType.primary_key: rendered_column_constraint = f"primary key {constraint_expression}" - elif constraint.type == ConstraintType.foreign_key and constraint_expression: - rendered_column_constraint = f"references {constraint_expression}" + elif constraint.type == ConstraintType.foreign_key: + if constraint.to and constraint.to_columns: + rendered_column_constraint = ( + f"references {constraint.to} ({', '.join(constraint.to_columns)})" + ) + elif constraint_expression: + rendered_column_constraint = f"references {constraint_expression}" elif constraint.type == ConstraintType.custom and constraint_expression: rendered_column_constraint = constraint_expression @@ -1670,20 +1687,29 @@ def render_model_constraint(cls, constraint: ModelLevelConstraint) -> Optional[s rendering.""" constraint_prefix = f"constraint {constraint.name} " if constraint.name else "" column_list = ", ".join(constraint.columns) + rendered_model_constraint = None + if constraint.type == ConstraintType.check and constraint.expression: - return f"{constraint_prefix}check ({constraint.expression})" + rendered_model_constraint = f"{constraint_prefix}check ({constraint.expression})" elif constraint.type == ConstraintType.unique: constraint_expression = f" {constraint.expression}" if constraint.expression else "" - return f"{constraint_prefix}unique{constraint_expression} ({column_list})" + rendered_model_constraint = ( + f"{constraint_prefix}unique{constraint_expression} ({column_list})" + ) elif constraint.type == ConstraintType.primary_key: constraint_expression = f" {constraint.expression}" if constraint.expression else "" - return f"{constraint_prefix}primary key{constraint_expression} ({column_list})" - elif constraint.type == ConstraintType.foreign_key and constraint.expression: - return f"{constraint_prefix}foreign key ({column_list}) references {constraint.expression}" + rendered_model_constraint = ( + f"{constraint_prefix}primary key{constraint_expression} ({column_list})" + ) + elif constraint.type == ConstraintType.foreign_key: + if constraint.to and constraint.to_columns: + rendered_model_constraint = f"{constraint_prefix}foreign key ({column_list}) references {constraint.to} ({', '.join(constraint.to_columns)})" + elif constraint.expression: + rendered_model_constraint = f"{constraint_prefix}foreign key ({column_list}) references {constraint.expression}" elif constraint.type == ConstraintType.custom and constraint.expression: - return f"{constraint_prefix}{constraint.expression}" - else: - return None + rendered_model_constraint = f"{constraint_prefix}{constraint.expression}" + + return rendered_model_constraint @classmethod def capabilities(cls) -> CapabilityDict: diff --git a/dbt/adapters/base/relation.py b/dbt/adapters/base/relation.py index 210a2dcd7..1aab7b2fe 100644 --- a/dbt/adapters/base/relation.py +++ b/dbt/adapters/base/relation.py @@ -241,8 +241,11 @@ def create_ephemeral_from( relation_config: RelationConfig, limit: Optional[int] = None, ) -> Self: - # Note that ephemeral models are based on the name. - identifier = cls.add_ephemeral_prefix(relation_config.name) + # Note that ephemeral models are based on the identifier, which will + # point to the model's alias if one exists and otherwise fall back to + # the filename. This is intended to give the user more control over + # the way that the CTE name is constructed + identifier = cls.add_ephemeral_prefix(relation_config.identifier) return cls.create( type=cls.CTE, identifier=identifier, diff --git a/dbt/adapters/capability.py b/dbt/adapters/capability.py index 305604c71..2bd491123 100644 --- a/dbt/adapters/capability.py +++ b/dbt/adapters/capability.py @@ -14,7 +14,12 @@ class Capability(str, Enum): """Indicates support for determining the time of the last table modification by querying database metadata.""" TableLastModifiedMetadataBatch = "TableLastModifiedMetadataBatch" - """Indicates support for performantly determining the time of the last table modification by querying database metadata in batch.""" + """Indicates support for performantly determining the time of the last table modification by querying database + metadata in batch.""" + + GetCatalogForSingleRelation = "GetCatalogForSingleRelation" + """Indicates support for getting catalog information including table-level and column-level metadata for a single + relation.""" class Support(str, Enum): diff --git a/dbt/adapters/events/types.py b/dbt/adapters/events/types.py index 7d98269d9..47c48da62 100644 --- a/dbt/adapters/events/types.py +++ b/dbt/adapters/events/types.py @@ -190,7 +190,7 @@ def code(self) -> str: return "E017" def message(self) -> str: - return f"SQL status: {self.status} in {self.elapsed} seconds" + return f"SQL status: {self.status} in {self.elapsed:.3f} seconds" class SQLCommit(DebugLevel): diff --git a/dbt/adapters/record.py b/dbt/adapters/record.py deleted file mode 100644 index 5204f59ce..000000000 --- a/dbt/adapters/record.py +++ /dev/null @@ -1,67 +0,0 @@ -import dataclasses -from io import StringIO -import json -import re -from typing import Any, Optional, Mapping - -from agate import Table - -from dbt_common.events.contextvars import get_node_info -from dbt_common.record import Record, Recorder - -from dbt.adapters.contracts.connection import AdapterResponse - - -@dataclasses.dataclass -class QueryRecordParams: - sql: str - auto_begin: bool = False - fetch: bool = False - limit: Optional[int] = None - node_unique_id: Optional[str] = None - - def __post_init__(self) -> None: - if self.node_unique_id is None: - node_info = get_node_info() - self.node_unique_id = node_info["unique_id"] if node_info else "" - - @staticmethod - def _clean_up_sql(sql: str) -> str: - sql = re.sub(r"--.*?\n", "", sql) # Remove single-line comments (--) - sql = re.sub(r"/\*.*?\*/", "", sql, flags=re.DOTALL) # Remove multi-line comments (/* */) - return sql.replace(" ", "").replace("\n", "") - - def _matches(self, other: "QueryRecordParams") -> bool: - return self.node_unique_id == other.node_unique_id and self._clean_up_sql( - self.sql - ) == self._clean_up_sql(other.sql) - - -@dataclasses.dataclass -class QueryRecordResult: - adapter_response: Optional["AdapterResponse"] - table: Optional[Table] - - def _to_dict(self) -> Any: - buf = StringIO() - self.table.to_json(buf) # type: ignore - - return { - "adapter_response": self.adapter_response.to_dict(), # type: ignore - "table": buf.getvalue(), - } - - @classmethod - def _from_dict(cls, dct: Mapping) -> "QueryRecordResult": - return QueryRecordResult( - adapter_response=AdapterResponse.from_dict(dct["adapter_response"]), - table=Table.from_object(json.loads(dct["table"])), - ) - - -class QueryRecord(Record): - params_cls = QueryRecordParams - result_cls = QueryRecordResult - - -Recorder.register_record_type(QueryRecord) diff --git a/dbt/adapters/record/__init__.py b/dbt/adapters/record/__init__.py new file mode 100644 index 000000000..afde4a01c --- /dev/null +++ b/dbt/adapters/record/__init__.py @@ -0,0 +1,2 @@ +from dbt.adapters.record.handle import RecordReplayHandle +from dbt.adapters.record.cursor.cursor import RecordReplayCursor diff --git a/dbt/adapters/record/cursor/cursor.py b/dbt/adapters/record/cursor/cursor.py new file mode 100644 index 000000000..577178dbb --- /dev/null +++ b/dbt/adapters/record/cursor/cursor.py @@ -0,0 +1,54 @@ +from typing import Any, Optional + +from dbt_common.record import record_function + +from dbt.adapters.contracts.connection import Connection +from dbt.adapters.record.cursor.description import CursorGetDescriptionRecord +from dbt.adapters.record.cursor.execute import CursorExecuteRecord +from dbt.adapters.record.cursor.fetchone import CursorFetchOneRecord +from dbt.adapters.record.cursor.fetchmany import CursorFetchManyRecord +from dbt.adapters.record.cursor.fetchall import CursorFetchAllRecord +from dbt.adapters.record.cursor.rowcount import CursorGetRowCountRecord + + +class RecordReplayCursor: + """A proxy object used to wrap native database cursors under record/replay + modes. In record mode, this proxy notes the parameters and return values + of the methods and properties it implements, which closely match the Python + DB API 2.0 cursor methods used by many dbt adapters to interact with the + database or DWH. In replay mode, it mocks out those calls using previously + recorded calls, so that no interaction with a database actually occurs.""" + + def __init__(self, native_cursor: Any, connection: Connection) -> None: + self.native_cursor = native_cursor + self.connection = connection + + @record_function(CursorExecuteRecord, method=True, id_field_name="connection_name") + def execute(self, operation, parameters=None) -> None: + self.native_cursor.execute(operation, parameters) + + @record_function(CursorFetchOneRecord, method=True, id_field_name="connection_name") + def fetchone(self) -> Any: + return self.native_cursor.fetchone() + + @record_function(CursorFetchManyRecord, method=True, id_field_name="connection_name") + def fetchmany(self, size: int) -> Any: + return self.native_cursor.fetchmany(size) + + @record_function(CursorFetchAllRecord, method=True, id_field_name="connection_name") + def fetchall(self) -> Any: + return self.native_cursor.fetchall() + + @property + def connection_name(self) -> Optional[str]: + return self.connection.name + + @property + @record_function(CursorGetRowCountRecord, method=True, id_field_name="connection_name") + def rowcount(self) -> int: + return self.native_cursor.rowcount + + @property + @record_function(CursorGetDescriptionRecord, method=True, id_field_name="connection_name") + def description(self) -> str: + return self.native_cursor.description diff --git a/dbt/adapters/record/cursor/description.py b/dbt/adapters/record/cursor/description.py new file mode 100644 index 000000000..d6ba15d97 --- /dev/null +++ b/dbt/adapters/record/cursor/description.py @@ -0,0 +1,37 @@ +import dataclasses +from typing import Any, Iterable, Mapping + +from dbt_common.record import Record, Recorder + + +@dataclasses.dataclass +class CursorGetDescriptionParams: + connection_name: str + + +@dataclasses.dataclass +class CursorGetDescriptionResult: + columns: Iterable[Any] + + def _to_dict(self) -> Any: + column_dicts = [] + for c in self.columns: + # This captures the mandatory column information, but we might need + # more for some adapters. + # See https://peps.python.org/pep-0249/#description + column_dicts.append((c[0], c[1])) + + return {"columns": column_dicts} + + @classmethod + def _from_dict(cls, dct: Mapping) -> "CursorGetDescriptionResult": + return CursorGetDescriptionResult(columns=dct["columns"]) + + +@Recorder.register_record_type +class CursorGetDescriptionRecord(Record): + """Implements record/replay support for the cursor.description property.""" + + params_cls = CursorGetDescriptionParams + result_cls = CursorGetDescriptionResult + group = "Database" diff --git a/dbt/adapters/record/cursor/execute.py b/dbt/adapters/record/cursor/execute.py new file mode 100644 index 000000000..e7e698591 --- /dev/null +++ b/dbt/adapters/record/cursor/execute.py @@ -0,0 +1,20 @@ +import dataclasses +from typing import Any, Iterable, Union, Mapping + +from dbt_common.record import Record, Recorder + + +@dataclasses.dataclass +class CursorExecuteParams: + connection_name: str + operation: str + parameters: Union[Iterable[Any], Mapping[str, Any]] + + +@Recorder.register_record_type +class CursorExecuteRecord(Record): + """Implements record/replay support for the cursor.execute() method.""" + + params_cls = CursorExecuteParams + result_cls = None + group = "Database" diff --git a/dbt/adapters/record/cursor/fetchall.py b/dbt/adapters/record/cursor/fetchall.py new file mode 100644 index 000000000..090cc1603 --- /dev/null +++ b/dbt/adapters/record/cursor/fetchall.py @@ -0,0 +1,66 @@ +import dataclasses +import datetime +from typing import Any, Dict, List, Mapping + +from dbt_common.record import Record, Recorder + + +@dataclasses.dataclass +class CursorFetchAllParams: + connection_name: str + + +@dataclasses.dataclass +class CursorFetchAllResult: + results: List[Any] + + def _to_dict(self) -> Dict[str, Any]: + processed_results = [] + for result in self.results: + result = tuple(map(self._process_value, result)) + processed_results.append(result) + + return {"results": processed_results} + + @classmethod + def _from_dict(cls, dct: Mapping) -> "CursorFetchAllResult": + unprocessed_results = [] + for result in dct["results"]: + result = tuple(map(cls._unprocess_value, result)) + unprocessed_results.append(result) + + return CursorFetchAllResult(unprocessed_results) + + @classmethod + def _process_value(cls, value: Any) -> Any: + if type(value) is datetime.date: + return {"type": "date", "value": value.isoformat()} + elif type(value) is datetime.datetime: + return {"type": "datetime", "value": value.isoformat()} + else: + return value + + @classmethod + def _unprocess_value(cls, value: Any) -> Any: + if type(value) is dict: + value_type = value.get("type") + if value_type == "date": + date_string = value.get("value") + assert isinstance(date_string, str) + return datetime.date.fromisoformat(date_string) + elif value_type == "datetime": + date_string = value.get("value") + assert isinstance(date_string, str) + return datetime.datetime.fromisoformat(date_string) + return value + else: + return value + + +@Recorder.register_record_type +class CursorFetchAllRecord(Record): + """Implements record/replay support for the cursor.fetchall() method.""" + + params_cls = CursorFetchAllParams + result_cls = CursorFetchAllResult + group = "Database" diff --git a/dbt/adapters/record/cursor/fetchmany.py b/dbt/adapters/record/cursor/fetchmany.py new file mode 100644 index 000000000..86f154408 --- /dev/null +++ b/dbt/adapters/record/cursor/fetchmany.py @@ -0,0 +1,23 @@ +import dataclasses +from typing import Any, List + +from dbt_common.record import Record, Recorder + + +@dataclasses.dataclass +class CursorFetchManyParams: + connection_name: str + + +@dataclasses.dataclass +class CursorFetchManyResult: + results: List[Any] + + +@Recorder.register_record_type +class CursorFetchManyRecord(Record): + """Implements record/replay support for the cursor.fetchmany() method.""" + + params_cls = CursorFetchManyParams + result_cls = CursorFetchManyResult + group = "Database" diff --git a/dbt/adapters/record/cursor/fetchone.py b/dbt/adapters/record/cursor/fetchone.py new file mode 100644 index 000000000..42ffe210c --- /dev/null +++ b/dbt/adapters/record/cursor/fetchone.py @@ -0,0 +1,23 @@ +import dataclasses +from typing import Any + +from dbt_common.record import Record, Recorder + + +@dataclasses.dataclass +class CursorFetchOneParams: + connection_name: str + + +@dataclasses.dataclass +class CursorFetchOneResult: + result: Any + + +@Recorder.register_record_type +class CursorFetchOneRecord(Record): + """Implements record/replay support for the cursor.fetchone() method.""" + + params_cls = CursorFetchOneParams + result_cls = CursorFetchOneResult + group = "Database" diff --git a/dbt/adapters/record/cursor/rowcount.py b/dbt/adapters/record/cursor/rowcount.py new file mode 100644 index 000000000..c024817ef --- /dev/null +++ b/dbt/adapters/record/cursor/rowcount.py @@ -0,0 +1,23 @@ +import dataclasses +from typing import Optional + +from dbt_common.record import Record, Recorder + + +@dataclasses.dataclass +class CursorGetRowCountParams: + connection_name: str + + +@dataclasses.dataclass +class CursorGetRowCountResult: + rowcount: Optional[int] + + +@Recorder.register_record_type +class CursorGetRowCountRecord(Record): + """Implements record/replay support for the cursor.rowcount property.""" + + params_cls = CursorGetRowCountParams + result_cls = CursorGetRowCountResult + group = "Database" diff --git a/dbt/adapters/record/handle.py b/dbt/adapters/record/handle.py new file mode 100644 index 000000000..31817c374 --- /dev/null +++ b/dbt/adapters/record/handle.py @@ -0,0 +1,24 @@ +from typing import Any + +from dbt.adapters.contracts.connection import Connection + +from dbt.adapters.record.cursor.cursor import RecordReplayCursor + + +class RecordReplayHandle: + """A proxy object used for record/replay modes. What adapters call a + 'handle' is typically a native database connection, but should not be + confused with the Connection protocol, which is a dbt-adapters concept. + + Currently, the only function of the handle proxy is to provide a record/replay + aware cursor object when cursor() is called.""" + + def __init__(self, native_handle: Any, connection: Connection) -> None: + self.native_handle = native_handle + self.connection = connection + + def cursor(self) -> Any: + # The native handle could be None if we are in replay mode, because no + # actual database access should be performed in that mode. + cursor = None if self.native_handle is None else self.native_handle.cursor() + return RecordReplayCursor(cursor, self.connection) diff --git a/dbt/adapters/sql/connections.py b/dbt/adapters/sql/connections.py index d8699fd39..4d450c88c 100644 --- a/dbt/adapters/sql/connections.py +++ b/dbt/adapters/sql/connections.py @@ -5,7 +5,6 @@ from dbt_common.events.contextvars import get_node_info from dbt_common.events.functions import fire_event from dbt_common.exceptions import DbtInternalError, NotImplementedError -from dbt_common.record import record_function from dbt_common.utils import cast_to_str from dbt.adapters.base import BaseConnectionManager @@ -20,7 +19,6 @@ SQLQuery, SQLQueryStatus, ) -from dbt.adapters.record import QueryRecord if TYPE_CHECKING: import agate @@ -88,7 +86,8 @@ def add_query( node_info=get_node_info(), ) ) - pre = time.time() + + pre = time.perf_counter() cursor = connection.handle.cursor() cursor.execute(sql, bindings) @@ -96,7 +95,7 @@ def add_query( fire_event( SQLQueryStatus( status=str(self.get_response(cursor)), - elapsed=round((time.time() - pre)), + elapsed=time.perf_counter() - pre, node_info=get_node_info(), ) ) @@ -142,7 +141,6 @@ def get_result_from_cursor(cls, cursor: Any, limit: Optional[int]) -> "agate.Tab return table_from_data_flat(data, column_names) - @record_function(QueryRecord, method=True, tuple_result=True) def execute( self, sql: str, diff --git a/dbt/adapters/sql/impl.py b/dbt/adapters/sql/impl.py index 8c6e0e8e4..8a8473f27 100644 --- a/dbt/adapters/sql/impl.py +++ b/dbt/adapters/sql/impl.py @@ -9,7 +9,6 @@ from dbt.adapters.exceptions import RelationTypeNullError from dbt.adapters.sql.connections import SQLConnectionManager - LIST_RELATIONS_MACRO_NAME = "list_relations_without_caching" GET_COLUMNS_IN_RELATION_MACRO_NAME = "get_columns_in_relation" LIST_SCHEMAS_MACRO_NAME = "list_schemas" @@ -41,6 +40,7 @@ class SQLAdapter(BaseAdapter): - get_catalog - list_relations_without_caching - get_columns_in_relation + - get_catalog_for_single_relation """ ConnectionManager: Type[SQLConnectionManager] diff --git a/dbt/include/global_project/macros/adapters/apply_grants.sql b/dbt/include/global_project/macros/adapters/apply_grants.sql index 10906e7ff..c75eef89d 100644 --- a/dbt/include/global_project/macros/adapters/apply_grants.sql +++ b/dbt/include/global_project/macros/adapters/apply_grants.sql @@ -61,7 +61,7 @@ {% endmacro %} {% macro default__get_show_grant_sql(relation) %} - show grants on {{ relation }} + show grants on {{ relation.render() }} {% endmacro %} @@ -70,7 +70,7 @@ {% endmacro %} {%- macro default__get_grant_sql(relation, privilege, grantees) -%} - grant {{ privilege }} on {{ relation }} to {{ grantees | join(', ') }} + grant {{ privilege }} on {{ relation.render() }} to {{ grantees | join(', ') }} {%- endmacro -%} @@ -79,7 +79,7 @@ {% endmacro %} {%- macro default__get_revoke_sql(relation, privilege, grantees) -%} - revoke {{ privilege }} on {{ relation }} from {{ grantees | join(', ') }} + revoke {{ privilege }} on {{ relation.render() }} from {{ grantees | join(', ') }} {%- endmacro -%} @@ -147,7 +147,7 @@ {% set needs_granting = diff_of_two_dicts(grant_config, current_grants_dict) %} {% set needs_revoking = diff_of_two_dicts(current_grants_dict, grant_config) %} {% if not (needs_granting or needs_revoking) %} - {{ log('On ' ~ relation ~': All grants are in place, no revocation or granting needed.')}} + {{ log('On ' ~ relation.render() ~': All grants are in place, no revocation or granting needed.')}} {% endif %} {% else %} {#-- We don't think there's any chance of previous grants having carried over. --#} diff --git a/dbt/include/global_project/macros/adapters/columns.sql b/dbt/include/global_project/macros/adapters/columns.sql index 663a827b1..96e6f3f24 100644 --- a/dbt/include/global_project/macros/adapters/columns.sql +++ b/dbt/include/global_project/macros/adapters/columns.sql @@ -96,10 +96,10 @@ {%- set tmp_column = column_name + "__dbt_alter" -%} {% call statement('alter_column_type') %} - alter table {{ relation }} add column {{ adapter.quote(tmp_column) }} {{ new_column_type }}; - update {{ relation }} set {{ adapter.quote(tmp_column) }} = {{ adapter.quote(column_name) }}; - alter table {{ relation }} drop column {{ adapter.quote(column_name) }} cascade; - alter table {{ relation }} rename column {{ adapter.quote(tmp_column) }} to {{ adapter.quote(column_name) }} + alter table {{ relation.render() }} add column {{ adapter.quote(tmp_column) }} {{ new_column_type }}; + update {{ relation.render() }} set {{ adapter.quote(tmp_column) }} = {{ adapter.quote(column_name) }}; + alter table {{ relation.render() }} drop column {{ adapter.quote(column_name) }} cascade; + alter table {{ relation.render() }} rename column {{ adapter.quote(tmp_column) }} to {{ adapter.quote(column_name) }} {% endcall %} {% endmacro %} @@ -120,7 +120,7 @@ {% set sql -%} - alter {{ relation.type }} {{ relation }} + alter {{ relation.type }} {{ relation.render() }} {% for column in add_columns %} add column {{ column.name }} {{ column.data_type }}{{ ',' if not loop.last }} diff --git a/dbt/include/global_project/macros/adapters/metadata.sql b/dbt/include/global_project/macros/adapters/metadata.sql index c8e8a4140..0aa7aabb4 100644 --- a/dbt/include/global_project/macros/adapters/metadata.sql +++ b/dbt/include/global_project/macros/adapters/metadata.sql @@ -77,6 +77,15 @@ 'list_relations_without_caching macro not implemented for adapter '+adapter.type()) }} {% endmacro %} +{% macro get_catalog_for_single_relation(relation) %} + {{ return(adapter.dispatch('get_catalog_for_single_relation', 'dbt')(relation)) }} +{% endmacro %} + +{% macro default__get_catalog_for_single_relation(relation) %} + {{ exceptions.raise_not_implemented( + 'get_catalog_for_single_relation macro not implemented for adapter '+adapter.type()) }} +{% endmacro %} + {% macro get_relations() %} {{ return(adapter.dispatch('get_relations', 'dbt')()) }} {% endmacro %} diff --git a/dbt/include/global_project/macros/adapters/relation.sql b/dbt/include/global_project/macros/adapters/relation.sql index 1c2bd8800..b9af49692 100644 --- a/dbt/include/global_project/macros/adapters/relation.sql +++ b/dbt/include/global_project/macros/adapters/relation.sql @@ -38,7 +38,7 @@ {% macro default__truncate_relation(relation) -%} {% call statement('truncate_relation') -%} - truncate table {{ relation }} + truncate table {{ relation.render() }} {%- endcall %} {% endmacro %} diff --git a/dbt/include/global_project/macros/adapters/show.sql b/dbt/include/global_project/macros/adapters/show.sql index 33a93f3db..3a5faa98a 100644 --- a/dbt/include/global_project/macros/adapters/show.sql +++ b/dbt/include/global_project/macros/adapters/show.sql @@ -1,22 +1,26 @@ +{# + We expect a syntax error if dbt show is invoked both with a --limit flag to show + and with a limit predicate embedded in its inline query. No special handling is + provided out-of-box. +#} {% macro get_show_sql(compiled_code, sql_header, limit) -%} - {%- if sql_header -%} + {%- if sql_header is not none -%} {{ sql_header }} - {%- endif -%} - {%- if limit is not none -%} + {%- endif %} {{ get_limit_subquery_sql(compiled_code, limit) }} - {%- else -%} - {{ compiled_code }} - {%- endif -%} {% endmacro %} -{% macro get_limit_subquery_sql(sql, limit) %} - {{ adapter.dispatch('get_limit_subquery_sql', 'dbt')(sql, limit) }} -{% endmacro %} +{# + Not necessarily a true subquery anymore. Now, merely a query subordinate + to the calling macro. +#} +{%- macro get_limit_subquery_sql(sql, limit) -%} + {{ adapter.dispatch('get_limit_sql', 'dbt')(sql, limit) }} +{%- endmacro -%} -{% macro default__get_limit_subquery_sql(sql, limit) %} - select * - from ( - {{ sql }} - ) as model_limit_subq - limit {{ limit }} +{% macro default__get_limit_sql(sql, limit) %} + {{ compiled_code }} + {% if limit is not none %} + limit {{ limit }} + {%- endif -%} {% endmacro %} diff --git a/dbt/include/global_project/macros/materializations/models/clone/clone.sql b/dbt/include/global_project/macros/materializations/models/clone/clone.sql index 01c8c3930..56d80082d 100644 --- a/dbt/include/global_project/macros/materializations/models/clone/clone.sql +++ b/dbt/include/global_project/macros/materializations/models/clone/clone.sql @@ -27,14 +27,14 @@ {%- set target_relation = this.incorporate(type='table') -%} {% if existing_relation is not none and not existing_relation.is_table %} - {{ log("Dropping relation " ~ existing_relation ~ " because it is of type " ~ existing_relation.type) }} + {{ log("Dropping relation " ~ existing_relation.render() ~ " because it is of type " ~ existing_relation.type) }} {{ drop_relation_if_exists(existing_relation) }} {% endif %} -- as a general rule, data platforms that can clone tables can also do atomic 'create or replace' {% call statement('main') %} {% if target_relation and defer_relation and target_relation == defer_relation %} - {{ log("Target relation and defer relation are the same, skipping clone for relation: " ~ target_relation) }} + {{ log("Target relation and defer relation are the same, skipping clone for relation: " ~ target_relation.render()) }} {% else %} {{ create_or_replace_clone(target_relation, defer_relation) }} {% endif %} diff --git a/dbt/include/global_project/macros/materializations/models/clone/create_or_replace_clone.sql b/dbt/include/global_project/macros/materializations/models/clone/create_or_replace_clone.sql index 204e9e874..cdb2559c6 100644 --- a/dbt/include/global_project/macros/materializations/models/clone/create_or_replace_clone.sql +++ b/dbt/include/global_project/macros/materializations/models/clone/create_or_replace_clone.sql @@ -3,5 +3,5 @@ {% endmacro %} {% macro default__create_or_replace_clone(this_relation, defer_relation) %} - create or replace table {{ this_relation }} clone {{ defer_relation }} + create or replace table {{ this_relation.render() }} clone {{ defer_relation.render() }} {% endmacro %} diff --git a/dbt/include/global_project/macros/materializations/models/incremental/incremental.sql b/dbt/include/global_project/macros/materializations/models/incremental/incremental.sql index e8ff5c1ea..f932751a4 100644 --- a/dbt/include/global_project/macros/materializations/models/incremental/incremental.sql +++ b/dbt/include/global_project/macros/materializations/models/incremental/incremental.sql @@ -39,9 +39,12 @@ {% set need_swap = true %} {% else %} {% do run_query(get_create_table_as_sql(True, temp_relation, sql)) %} - {% do adapter.expand_target_column_types( - from_relation=temp_relation, - to_relation=target_relation) %} + {% set contract_config = config.get('contract') %} + {% if not contract_config or not contract_config.enforced %} + {% do adapter.expand_target_column_types( + from_relation=temp_relation, + to_relation=target_relation) %} + {% endif %} {#-- Process schema changes. Returns dict of changes if successful. Use source columns for upserting/merging --#} {% set dest_columns = process_schema_changes(on_schema_change, temp_relation, existing_relation) %} {% if not dest_columns %} diff --git a/dbt/include/global_project/macros/materializations/models/materialized_view.sql b/dbt/include/global_project/macros/materializations/models/materialized_view.sql index 6dc30bf9a..a39f8aa21 100644 --- a/dbt/include/global_project/macros/materializations/models/materialized_view.sql +++ b/dbt/include/global_project/macros/materializations/models/materialized_view.sql @@ -71,9 +71,9 @@ {% set build_sql = get_alter_materialized_view_as_sql(target_relation, configuration_changes, sql, existing_relation, backup_relation, intermediate_relation) %} {% elif on_configuration_change == 'continue' %} {% set build_sql = '' %} - {{ exceptions.warn("Configuration changes were identified and `on_configuration_change` was set to `continue` for `" ~ target_relation ~ "`") }} + {{ exceptions.warn("Configuration changes were identified and `on_configuration_change` was set to `continue` for `" ~ target_relation.render() ~ "`") }} {% elif on_configuration_change == 'fail' %} - {{ exceptions.raise_fail_fast_error("Configuration changes were identified and `on_configuration_change` was set to `fail` for `" ~ target_relation ~ "`") }} + {{ exceptions.raise_fail_fast_error("Configuration changes were identified and `on_configuration_change` was set to `fail` for `" ~ target_relation.render() ~ "`") }} {% else %} -- this only happens if the user provides a value other than `apply`, 'skip', 'fail' diff --git a/dbt/include/global_project/macros/materializations/seeds/helpers.sql b/dbt/include/global_project/macros/materializations/seeds/helpers.sql index 44dbf370d..d87c258b1 100644 --- a/dbt/include/global_project/macros/materializations/seeds/helpers.sql +++ b/dbt/include/global_project/macros/materializations/seeds/helpers.sql @@ -37,7 +37,7 @@ {% set sql = create_csv_table(model, agate_table) %} {% else %} {{ adapter.truncate_relation(old_relation) }} - {% set sql = "truncate table " ~ old_relation %} + {% set sql = "truncate table " ~ old_relation.render() %} {% endif %} {{ return(sql) }} diff --git a/dbt/include/global_project/macros/materializations/seeds/seed.sql b/dbt/include/global_project/macros/materializations/seeds/seed.sql index 3b66252da..4ee4fb80c 100644 --- a/dbt/include/global_project/macros/materializations/seeds/seed.sql +++ b/dbt/include/global_project/macros/materializations/seeds/seed.sql @@ -22,7 +22,7 @@ -- build model {% set create_table_sql = "" %} {% if exists_as_view %} - {{ exceptions.raise_compiler_error("Cannot seed to '{}', it is a view".format(old_relation)) }} + {{ exceptions.raise_compiler_error("Cannot seed to '{}', it is a view".format(old_relation.render())) }} {% elif exists_as_table %} {% set create_table_sql = reset_csv_table(model, full_refresh_mode, old_relation, agate_table) %} {% else %} diff --git a/dbt/include/global_project/macros/materializations/snapshots/helpers.sql b/dbt/include/global_project/macros/materializations/snapshots/helpers.sql index 7fd4bfd51..bb71974cf 100644 --- a/dbt/include/global_project/macros/materializations/snapshots/helpers.sql +++ b/dbt/include/global_project/macros/materializations/snapshots/helpers.sql @@ -8,7 +8,7 @@ {% macro default__create_columns(relation, columns) %} {% for column in columns %} {% call statement() %} - alter table {{ relation }} add column "{{ column.name }}" {{ column.data_type }}; + alter table {{ relation.render() }} add column "{{ column.name }}" {{ column.data_type }}; {% endcall %} {% endfor %} {% endmacro %} diff --git a/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql b/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql index 6bc50fd3b..56798811d 100644 --- a/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql +++ b/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql @@ -7,7 +7,7 @@ {% macro default__snapshot_merge_sql(target, source, insert_cols) -%} {%- set insert_cols_csv = insert_cols | join(', ') -%} - merge into {{ target }} as DBT_INTERNAL_DEST + merge into {{ target.render() }} as DBT_INTERNAL_DEST using {{ source }} as DBT_INTERNAL_SOURCE on DBT_INTERNAL_SOURCE.dbt_scd_id = DBT_INTERNAL_DEST.dbt_scd_id diff --git a/dbt/include/global_project/macros/relations/drop.sql b/dbt/include/global_project/macros/relations/drop.sql index 58abd14d9..e66511dab 100644 --- a/dbt/include/global_project/macros/relations/drop.sql +++ b/dbt/include/global_project/macros/relations/drop.sql @@ -16,7 +16,7 @@ {{ drop_materialized_view(relation) }} {%- else -%} - drop {{ relation.type }} if exists {{ relation }} cascade + drop {{ relation.type }} if exists {{ relation.render() }} cascade {%- endif -%} diff --git a/dbt/include/global_project/macros/relations/materialized_view/drop.sql b/dbt/include/global_project/macros/relations/materialized_view/drop.sql index b218d0f3c..8235b1c69 100644 --- a/dbt/include/global_project/macros/relations/materialized_view/drop.sql +++ b/dbt/include/global_project/macros/relations/materialized_view/drop.sql @@ -10,5 +10,5 @@ actually executes the drop, and `get_drop_sql`, which returns the template. {% macro default__drop_materialized_view(relation) -%} - drop materialized view if exists {{ relation }} cascade + drop materialized view if exists {{ relation.render() }} cascade {%- endmacro %} diff --git a/dbt/include/global_project/macros/relations/rename.sql b/dbt/include/global_project/macros/relations/rename.sql index d7f3a72e2..4b913df35 100644 --- a/dbt/include/global_project/macros/relations/rename.sql +++ b/dbt/include/global_project/macros/relations/rename.sql @@ -30,6 +30,6 @@ {% macro default__rename_relation(from_relation, to_relation) -%} {% set target_name = adapter.quote_as_configured(to_relation.identifier, 'identifier') %} {% call statement('rename_relation') -%} - alter table {{ from_relation }} rename to {{ target_name }} + alter table {{ from_relation.render() }} rename to {{ target_name }} {%- endcall %} {% endmacro %} diff --git a/dbt/include/global_project/macros/relations/table/drop.sql b/dbt/include/global_project/macros/relations/table/drop.sql index d7d5941c4..038ded9ea 100644 --- a/dbt/include/global_project/macros/relations/table/drop.sql +++ b/dbt/include/global_project/macros/relations/table/drop.sql @@ -10,5 +10,5 @@ actually executes the drop, and `get_drop_sql`, which returns the template. {% macro default__drop_table(relation) -%} - drop table if exists {{ relation }} cascade + drop table if exists {{ relation.render() }} cascade {%- endmacro %} diff --git a/dbt/include/global_project/macros/relations/view/create.sql b/dbt/include/global_project/macros/relations/view/create.sql index 41cd196c3..ee83befae 100644 --- a/dbt/include/global_project/macros/relations/view/create.sql +++ b/dbt/include/global_project/macros/relations/view/create.sql @@ -16,7 +16,7 @@ {%- set sql_header = config.get('sql_header', none) -%} {{ sql_header if sql_header is not none }} - create view {{ relation }} + create view {{ relation.render() }} {% set contract_config = config.get('contract') %} {% if contract_config.enforced %} {{ get_assert_columns_equivalent(sql) }} diff --git a/dbt/include/global_project/macros/relations/view/drop.sql b/dbt/include/global_project/macros/relations/view/drop.sql index 7e1924fae..84c91a364 100644 --- a/dbt/include/global_project/macros/relations/view/drop.sql +++ b/dbt/include/global_project/macros/relations/view/drop.sql @@ -10,5 +10,5 @@ actually executes the drop, and `get_drop_sql`, which returns the template. {% macro default__drop_view(relation) -%} - drop view if exists {{ relation }} cascade + drop view if exists {{ relation.render() }} cascade {%- endmacro %} diff --git a/dbt/include/global_project/macros/relations/view/replace.sql b/dbt/include/global_project/macros/relations/view/replace.sql index 1da061347..a0f0dc76f 100644 --- a/dbt/include/global_project/macros/relations/view/replace.sql +++ b/dbt/include/global_project/macros/relations/view/replace.sql @@ -61,6 +61,6 @@ {% endmacro %} {% macro default__handle_existing_table(full_refresh, old_relation) %} - {{ log("Dropping relation " ~ old_relation ~ " because it is of type " ~ old_relation.type) }} + {{ log("Dropping relation " ~ old_relation.render() ~ " because it is of type " ~ old_relation.type) }} {{ adapter.drop_relation(old_relation) }} {% endmacro %} diff --git a/dbt/include/global_project/macros/unit_test_sql/get_fixture_sql.sql b/dbt/include/global_project/macros/unit_test_sql/get_fixture_sql.sql index c25a87f7b..a3a8173ba 100644 --- a/dbt/include/global_project/macros/unit_test_sql/get_fixture_sql.sql +++ b/dbt/include/global_project/macros/unit_test_sql/get_fixture_sql.sql @@ -22,6 +22,7 @@ {%- do default_row.update({column_name: (safe_cast("null", column_type) | trim )}) -%} {%- endfor -%} +{{ validate_fixture_rows(rows, row_number) }} {%- for row in rows -%} {%- set formatted_row = format_row(row, column_name_to_data_types) -%} @@ -93,3 +94,11 @@ union all {%- endfor -%} {{ return(formatted_row) }} {%- endmacro -%} + +{%- macro validate_fixture_rows(rows, row_number) -%} + {{ return(adapter.dispatch('validate_fixture_rows', 'dbt')(rows, row_number)) }} +{%- endmacro -%} + +{%- macro default__validate_fixture_rows(rows, row_number) -%} + {# This is an abstract method for adapter overrides as needed #} +{%- endmacro -%} diff --git a/docs/guides/record_replay.md b/docs/guides/record_replay.md index 670bb8431..5bcbec06c 100644 --- a/docs/guides/record_replay.md +++ b/docs/guides/record_replay.md @@ -4,24 +4,12 @@ This document describes how to implement support for dbt's Record/Replay Subsyst ## Recording and Replaying Warehouse Interaction -The goal of the Record/Replay Subsystem is to record all interactions between dbt and external systems, of which the data warehouse is the most obvious. Since, warehouse interaction is mediated by adapters, full Record/Replay support requires that adapters record all interactions they have with the warehouse. (It also requires that they record access to the local filesystem or external service, if that is access is not mediated by dbt itself. This includes authentication steps, opening and closing connections, beginning and ending transactions, and so forth.) +The goal of the Record/Replay Subsystem is to record all interactions between dbt and external systems, of which the data warehouse is the most important. Since, warehouse interaction is mediated by adapters, full Record/Replay support requires that adapters record all interactions they have with the warehouse. It also requires that they record access to the local filesystem or external service, if that is access is not mediated by dbt itself. This includes authentication steps, opening and closing connections, beginning and ending transactions, etc. -In practice, this means that any request sent to the warehouse must be recorded, along with the corresponding response. If this is done correctly, as described in the document linked in the intro, the Record portion of the Record/Replay subsystem should work as expected. - -At the time of this writing, there is only an incomplete implementation of this goal, which can be found in `dbt-adapters/dbt/adapters/record.py`. - -There are some important things to notice about this implementation. First, the QueryRecordResult class provides custom serialization methods `to_dict()` and `from_dict()`. This is necessary because the `AdapterResponse` and `Agate.Table` types cannot be automatically converted to and from JSON by the dataclass library, and JSON is the format used to persist recordings to disk and reload them for replay. - -Another important feature is that `QueryRecordParams` implements the `_matches()` method. This method allows `dbt-adapters` to customize the way that the Record/Replay determines whether a query issued by dbt matches a previously recorded query. In this case, the method performs a comparison which attempts to ignore comments and whitespace which would not affect query behavior. +A basic implementation of Record/Replay functionality, suitable for most adapters which extend the `SQLAdapter` class, can be found in `dbt-adapters/dbt/adapters/record`. The `RecordReplayHandle` and `RecordReplayCursor` classes defined there are used to intercept and record or replay all DWH interactions. They are an excellent starting point for adapters which extend `SQLAdapter` and use a database library which substantially conforms to Python's DB API v2.0 (PEP 249). Examples of how library-specific deviations from that API can be found in the dbt-postgress and dbt-snowflake repositories. ## Misc. Notes and Suggestions -Currently, support for recording data warehouse interaction is very rudimentary, however, even rudimentary support is valuable and we should be concentrating on extending it in a way that adds the most value with the least work. Usefulness, rather than perfection, is the initial goal. - -Picking the right functions to record, at the right level of abstraction, will probably be the most important part of carrying this work forward. - Not every interaction with an external system has to be recorded in full detail, and authentication might prove to be a place where we exclude sensitive secrets from the recording. For example, since replay will not actually be communicating with the warehouse, it may be possible to exclude passwords and auth keys from the parameters recorded, and to exclude auth tokens from the results. In addition to adding an appropriate decorator to functions which communicate with external systems, you should check those functions for side-effects. Since the function's calls will be mocked out in replay mode, those side-effects will not be carried out during replay. At present, we are focusing on support for recording and comparing recordings, but this is worth keeping in mind. - -The current implementation records which dbt node issues a query, and uses that information to ensure a match during replay. The same node should issue the same query. A better model might be to monitor which connection issued which query, and associate the same connection with open/close operations, transaction starts/stops and so forth. diff --git a/pyproject.toml b/pyproject.toml index a4b011a8f..e794781c8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ maintainers = [ { name = "dbt Labs", email = "info@dbtlabs.com" }, ] classifiers = [ - "Development Status :: 2 - Pre-Alpha", + "Development Status :: 5 - Production/Stable", "License :: OSI Approved :: Apache Software License", "Operating System :: MacOS :: MacOS X", "Operating System :: Microsoft :: Windows", @@ -24,7 +24,7 @@ classifiers = [ "Programming Language :: Python :: 3.12", ] dependencies = [ - "dbt-common<2.0", + "dbt-common>=1.6,<2.0", "pytz>=2015.7", # installed via dbt-common but used directly "agate>=1.0,<2.0", @@ -57,20 +57,14 @@ dependencies = [ "dbt_common @ git+https://github.com/dbt-labs/dbt-common.git", 'pre-commit==3.7.0;python_version>="3.9"', 'pre-commit==3.5.0;python_version=="3.8"', -] -[tool.hatch.envs.default.scripts] -dev = "pre-commit install" -code-quality = "pre-commit run --all-files" - -[tool.hatch.envs.unit-tests] -dependencies = [ - "dbt_common @ git+https://github.com/dbt-labs/dbt-common.git", "pytest", "pytest-dotenv", "pytest-xdist", ] -[tool.hatch.envs.unit-tests.scripts] -all = "python -m pytest {args:tests/unit}" +[tool.hatch.envs.default.scripts] +setup = "pre-commit install" +code-quality = "pre-commit run --all-files" +unit-tests = "python -m pytest {args:tests/unit}" [tool.hatch.envs.build] detached = true diff --git a/tests/unit/test_base_adapter.py b/tests/unit/test_base_adapter.py index 95fe5ae2b..5fa109b7e 100644 --- a/tests/unit/test_base_adapter.py +++ b/tests/unit/test_base_adapter.py @@ -39,6 +39,14 @@ def connection_manager(self): [{"type": "foreign_key", "expression": "other_table (c1)"}], ["column_name integer references other_table (c1)"], ), + ( + [{"type": "foreign_key", "to": "other_table", "to_columns": ["c1"]}], + ["column_name integer references other_table (c1)"], + ), + ( + [{"type": "foreign_key", "to": "other_table", "to_columns": ["c1", "c2"]}], + ["column_name integer references other_table (c1, c2)"], + ), ([{"type": "check"}, {"type": "unique"}], ["column_name integer unique"]), ([{"type": "custom", "expression": "-- noop"}], ["column_name integer -- noop"]), ] @@ -176,6 +184,30 @@ def test_render_raw_columns_constraints_unsupported( ], ["constraint test_name foreign key (c1, c2) references other_table (c1)"], ), + ( + [ + { + "type": "foreign_key", + "columns": ["c1", "c2"], + "to": "other_table", + "to_columns": ["c1"], + "name": "test_name", + } + ], + ["constraint test_name foreign key (c1, c2) references other_table (c1)"], + ), + ( + [ + { + "type": "foreign_key", + "columns": ["c1", "c2"], + "to": "other_table", + "to_columns": ["c1", "c2"], + "name": "test_name", + } + ], + ["constraint test_name foreign key (c1, c2) references other_table (c1, c2)"], + ), ] @pytest.mark.parametrize("constraints,expected_rendered_constraints", model_constraints) diff --git a/tests/unit/test_relation.py b/tests/unit/test_relation.py index a1c01c5c1..97d564192 100644 --- a/tests/unit/test_relation.py +++ b/tests/unit/test_relation.py @@ -1,4 +1,4 @@ -from dataclasses import replace +from dataclasses import dataclass, replace import pytest @@ -79,3 +79,16 @@ def test_render_limited(limit, require_alias, expected_result): actual_result = my_relation.render_limited() assert actual_result == expected_result assert str(my_relation) == expected_result + + +def test_create_ephemeral_from_uses_identifier(): + @dataclass + class Node: + """Dummy implementation of RelationConfig protocol""" + + name: str + identifier: str + + node = Node(name="name_should_not_be_used", identifier="test") + ephemeral_relation = BaseRelation.create_ephemeral_from(node) + assert str(ephemeral_relation) == "__dbt__cte__test"