From 03467bef28ce3a16c663f089a07806e663526428 Mon Sep 17 00:00:00 2001 From: error418 Date: Mon, 8 May 2023 21:03:17 +0200 Subject: [PATCH 1/7] feat(unpivot): add quote identifier parameter to unpivot macro fixes #216 --- CHANGELOG.md | 3 ++- README.md | 1 + .../data/sql/data_unpivot_quote.csv | 4 ++++ .../data/sql/data_unpivot_quote_expected.csv | 7 +++++++ integration_tests/dbt_project.yml | 5 +++++ integration_tests/models/sql/schema.yml | 6 ++++++ .../models/sql/test_unpivot_quote.sql | 18 ++++++++++++++++++ macros/sql/unpivot.sql | 17 +++++++++-------- 8 files changed, 52 insertions(+), 9 deletions(-) create mode 100644 integration_tests/data/sql/data_unpivot_quote.csv create mode 100644 integration_tests/data/sql/data_unpivot_quote_expected.csv create mode 100644 integration_tests/models/sql/test_unpivot_quote.sql diff --git a/CHANGELOG.md b/CHANGELOG.md index b70c348a..675009d3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,12 +10,13 @@ # Unreleased ## New features -* ZZZ by @YYY in https://github.com/dbt-labs/dbt-utils/pull/XXX +- Add `quote_identifiers` parameter to `unpivot` macro to handle case sensitive columns. This parameter defaults to `False` by @error418 in https://github.com/dbt-labs/dbt-utils/pull/792 ## Fixes * Fix legacy links in README by @dbeatty10 in https://github.com/dbt-labs/dbt-utils/pull/796 ## Quality of life ## Under the hood ## Contributors: +- [@error418] (https://github.com/error418) (#216) # dbt utils v1.1.0 ## What's Changed diff --git a/README.md b/README.md index 827626ba..22b30f50 100644 --- a/README.md +++ b/README.md @@ -1210,6 +1210,7 @@ Boolean values are replaced with the strings 'true'|'false' - `remove`: A list of columns to remove from the resulting table. - `field_name`: column name in the resulting table for field - `value_name`: column name in the resulting table for value +- `quote_identifiers` (optional, default=`False`): will encase selected columns and aliases in quotes according to your adapter's implementation of `adapter.quote` (e.g. `"field_name" as "field_name"`). ### width_bucket ([source](macros/sql/width_bucket.sql)) diff --git a/integration_tests/data/sql/data_unpivot_quote.csv b/integration_tests/data/sql/data_unpivot_quote.csv new file mode 100644 index 00000000..958e05f3 --- /dev/null +++ b/integration_tests/data/sql/data_unpivot_quote.csv @@ -0,0 +1,4 @@ +customer_id,created_at,sTaTuS,SEGMENT,name +123,2017-01-01,active,tier 1,name 1 +234,2017-02-01,active,tier 3,name 3 +567,2017-03-01,churned,tier 2,name 2 diff --git a/integration_tests/data/sql/data_unpivot_quote_expected.csv b/integration_tests/data/sql/data_unpivot_quote_expected.csv new file mode 100644 index 00000000..851ca429 --- /dev/null +++ b/integration_tests/data/sql/data_unpivot_quote_expected.csv @@ -0,0 +1,7 @@ +customer_id,created_at,prop,val +123,"2017-01-01","SEGMENT","tier 1" +123,"2017-01-01","sTaTuS","active" +234,"2017-02-01","SEGMENT","tier 3" +234,"2017-02-01","sTaTuS","active" +567,"2017-03-01","sTaTuS","churned" +567,"2017-03-01","SEGMENT","tier 2" diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index a9531e78..27ad0ee7 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -48,6 +48,11 @@ seeds: num_buckets: integer min_value: float max_value: float + + data_unpivot_quote: + +quote_columns: true + data_unpivot_quote_expected: + +quote_columns: true schema_tests: data_test_sequential_timestamps: diff --git a/integration_tests/models/sql/schema.yml b/integration_tests/models/sql/schema.yml index e79e782f..a5ebf80e 100644 --- a/integration_tests/models/sql/schema.yml +++ b/integration_tests/models/sql/schema.yml @@ -158,6 +158,12 @@ models: - dbt_utils.equality: compare_model: ref('data_unpivot_bool_expected') + - name: test_unpivot_quote + tests: + - dbt_utils.equality: + enabled: "{{ target.name != 'redshift' }}" # redshift lowercases all columns + compare_model: ref('data_unpivot_quote_expected') + - name: test_star tests: - dbt_utils.equality: diff --git a/integration_tests/models/sql/test_unpivot_quote.sql b/integration_tests/models/sql/test_unpivot_quote.sql new file mode 100644 index 00000000..7e480fe9 --- /dev/null +++ b/integration_tests/models/sql/test_unpivot_quote.sql @@ -0,0 +1,18 @@ +select + -- snowflake requires quoted column names for this test + {{ adapter.quote("customer_id") }}, + {{ adapter.quote("created_at") }}, + {{ adapter.quote("prop") }}, + {{ adapter.quote("val") }} + +from ( + {{ dbt_utils.unpivot( + relation=ref('data_unpivot_quote'), + cast_to=type_string(), + exclude=['customer_id', 'created_at'], + remove=['name'], + field_name='prop', + value_name='val', + quote_identifiers=True, + ) }} +) as sbq diff --git a/macros/sql/unpivot.sql b/macros/sql/unpivot.sql index 371b314b..c82b45e0 100644 --- a/macros/sql/unpivot.sql +++ b/macros/sql/unpivot.sql @@ -12,11 +12,11 @@ Arguments: value_name: Destination table column name for the pivoted values #} -{% macro unpivot(relation=none, cast_to='varchar', exclude=none, remove=none, field_name='field_name', value_name='value') -%} - {{ return(adapter.dispatch('unpivot', 'dbt_utils')(relation, cast_to, exclude, remove, field_name, value_name)) }} +{% macro unpivot(relation=none, cast_to='varchar', exclude=none, remove=none, field_name='field_name', value_name='value', quote_identifiers=False) -%} + {{ return(adapter.dispatch('unpivot', 'dbt_utils')(relation, cast_to, exclude, remove, field_name, value_name, quote_identifiers)) }} {% endmacro %} -{% macro default__unpivot(relation=none, cast_to='varchar', exclude=none, remove=none, field_name='field_name', value_name='value') -%} +{% macro default__unpivot(relation=none, cast_to='varchar', exclude=none, remove=none, field_name='field_name', value_name='value', quote_identifiers=False) -%} {% if not relation %} {{ exceptions.raise_compiler_error("Error: argument `relation` is required for `unpivot` macro.") }} @@ -43,18 +43,19 @@ Arguments: {%- for col in include_cols -%} + {%- set current_col_name = adapter.quote(col.column) if quote_identifiers else col.column -%} select {%- for exclude_col in exclude %} - {{ exclude_col }}, + {{ adapter.quote(exclude_col) if quote_identifiers else exclude_col }}, {%- endfor %} - cast('{{ col.column }}' as {{ dbt.type_string() }}) as {{ field_name }}, + cast('{{ col.column }}' as {{ dbt.type_string() }}) as {{ adapter.quote(field_name) if quote_identifiers else field_name }}, cast( {% if col.data_type == 'boolean' %} - {{ dbt.cast_bool_to_text(col.column) }} + {{ dbt.cast_bool_to_text(current_col_name) }} {% else %} - {{ col.column }} + {{ current_col_name }} {% endif %} - as {{ cast_to }}) as {{ value_name }} + as {{ cast_to }}) as {{ adapter.quote(value_name) if quote_identifiers else value_name }} from {{ relation }} From 6bf6b1108a9ce84ad2935d7479f9c79f2e916479 Mon Sep 17 00:00:00 2001 From: Doug Beatty Date: Tue, 25 Jun 2024 17:45:04 -0600 Subject: [PATCH 2/7] Changelog will be updated as part of release process --- CHANGELOG.md | 8 -------- 1 file changed, 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a6e60b83..16e296f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,18 +48,10 @@ # dbt utils v1.1.1 ## New features -* Add `quote_identifiers` parameter to `unpivot` macro to handle case sensitive columns. This parameter defaults to `False` by @error418 in https://github.com/dbt-labs/dbt-utils/pull/792 * Improve the performance of the `at_least_one` test by pruning early. This is especially helpful when running against external tables. By @joshuahuntley in https://github.com/dbt-labs/dbt-utils/pull/775 - ## Fixes * Fix legacy links in README by @dbeatty10 in https://github.com/dbt-labs/dbt-utils/pull/796 -## Quality of life -## Under the hood -## Contributors: -- [@error418] (https://github.com/error418) (#216) - - # dbt utils v1.1.0 ## What's Changed ### New functionality From 07ba6a42e912e1a2a19071b4a691f830b18e7528 Mon Sep 17 00:00:00 2001 From: Doug Beatty Date: Tue, 25 Jun 2024 17:49:24 -0600 Subject: [PATCH 3/7] Enable test for unpivoting quoted columns for Redshift --- integration_tests/models/sql/schema.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/integration_tests/models/sql/schema.yml b/integration_tests/models/sql/schema.yml index a6838c98..b3d531fc 100644 --- a/integration_tests/models/sql/schema.yml +++ b/integration_tests/models/sql/schema.yml @@ -161,7 +161,6 @@ models: - name: test_unpivot_quote tests: - dbt_utils.equality: - enabled: "{{ target.name != 'redshift' }}" # redshift lowercases all columns compare_model: ref('data_unpivot_quote_expected') - name: test_star From 3e7b4ee3d4a023c371ced6fee35741d9f4e4cb82 Mon Sep 17 00:00:00 2001 From: Doug Beatty Date: Wed, 26 Jun 2024 13:13:37 -0600 Subject: [PATCH 4/7] Fix warning related to `tests:` -> `data_tests:` --- integration_tests/models/sql/schema.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/models/sql/schema.yml b/integration_tests/models/sql/schema.yml index b3d531fc..850e4549 100644 --- a/integration_tests/models/sql/schema.yml +++ b/integration_tests/models/sql/schema.yml @@ -159,7 +159,7 @@ models: compare_model: ref('data_unpivot_bool_expected') - name: test_unpivot_quote - tests: + data_tests: - dbt_utils.equality: compare_model: ref('data_unpivot_quote_expected') From 02ba414ce90d206d4e5091c6d73dac02da0aa77e Mon Sep 17 00:00:00 2001 From: Doug Beatty Date: Wed, 26 Jun 2024 13:18:29 -0600 Subject: [PATCH 5/7] Simplify the case-sensitive unpivot test model --- integration_tests/models/sql/test_unpivot_quote.sql | 8 -------- 1 file changed, 8 deletions(-) diff --git a/integration_tests/models/sql/test_unpivot_quote.sql b/integration_tests/models/sql/test_unpivot_quote.sql index 7e480fe9..d5fdfcf1 100644 --- a/integration_tests/models/sql/test_unpivot_quote.sql +++ b/integration_tests/models/sql/test_unpivot_quote.sql @@ -1,11 +1,4 @@ -select - -- snowflake requires quoted column names for this test - {{ adapter.quote("customer_id") }}, - {{ adapter.quote("created_at") }}, - {{ adapter.quote("prop") }}, - {{ adapter.quote("val") }} -from ( {{ dbt_utils.unpivot( relation=ref('data_unpivot_quote'), cast_to=type_string(), @@ -15,4 +8,3 @@ from ( value_name='val', quote_identifiers=True, ) }} -) as sbq From cf6bfd88c42eb95700bcd3a2c6a0d6c2b1543705 Mon Sep 17 00:00:00 2001 From: Doug Beatty Date: Wed, 26 Jun 2024 14:25:15 -0600 Subject: [PATCH 6/7] Add a newline for consistency --- integration_tests/dbt_project.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 3b6dae5d..560efee3 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -51,6 +51,7 @@ seeds: data_unpivot_quote: +quote_columns: true + data_unpivot_quote_expected: +quote_columns: true From aa75ad7f23df277c0b70ce95492bf753e0cf8004 Mon Sep 17 00:00:00 2001 From: Doug Beatty Date: Wed, 26 Jun 2024 14:49:32 -0600 Subject: [PATCH 7/7] Use mixed-case column names for all columns --- integration_tests/data/sql/data_unpivot_quote.csv | 2 +- .../data/sql/data_unpivot_quote_expected.csv | 2 +- integration_tests/models/sql/test_unpivot_quote.sql | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/integration_tests/data/sql/data_unpivot_quote.csv b/integration_tests/data/sql/data_unpivot_quote.csv index 958e05f3..a5138d9f 100644 --- a/integration_tests/data/sql/data_unpivot_quote.csv +++ b/integration_tests/data/sql/data_unpivot_quote.csv @@ -1,4 +1,4 @@ -customer_id,created_at,sTaTuS,SEGMENT,name +Customer_Id,Created_At,sTaTuS,SEGMENT,Name 123,2017-01-01,active,tier 1,name 1 234,2017-02-01,active,tier 3,name 3 567,2017-03-01,churned,tier 2,name 2 diff --git a/integration_tests/data/sql/data_unpivot_quote_expected.csv b/integration_tests/data/sql/data_unpivot_quote_expected.csv index 851ca429..0704f1f8 100644 --- a/integration_tests/data/sql/data_unpivot_quote_expected.csv +++ b/integration_tests/data/sql/data_unpivot_quote_expected.csv @@ -1,4 +1,4 @@ -customer_id,created_at,prop,val +Customer_Id,Created_At,Prop,Val 123,"2017-01-01","SEGMENT","tier 1" 123,"2017-01-01","sTaTuS","active" 234,"2017-02-01","SEGMENT","tier 3" diff --git a/integration_tests/models/sql/test_unpivot_quote.sql b/integration_tests/models/sql/test_unpivot_quote.sql index d5fdfcf1..feaac6af 100644 --- a/integration_tests/models/sql/test_unpivot_quote.sql +++ b/integration_tests/models/sql/test_unpivot_quote.sql @@ -2,9 +2,9 @@ {{ dbt_utils.unpivot( relation=ref('data_unpivot_quote'), cast_to=type_string(), - exclude=['customer_id', 'created_at'], - remove=['name'], - field_name='prop', - value_name='val', + exclude=['Customer_Id', 'Created_At'], + remove=['Name'], + field_name='Prop', + value_name='Val', quote_identifiers=True, ) }}