Merge branch 'main' into develop

kgmcquate · Feb 28, 2024 · 8a643de · 8a643de
2 parents 7cf726e + 8729d3b
commit 8a643de
Show file tree

Hide file tree

Showing 15 changed files with 177 additions and 39 deletions.
diff --git a/.github/workflows/static.yml b/.github/workflows/static.yml
@@ -49,13 +49,13 @@ jobs:
 
       - name: Generate DBT docs
         # Copies to root /docs folder
-        run: bash generate_docs.sh
+        run: bash docs/generate_docs.sh
 
       - name: Upload artifact
         uses: actions/upload-pages-artifact@v3
         with:
           # Upload entire repository
-          path: '../docs/'
+          path: integration_tests/static/
       - name: Deploy to GitHub Pages
         id: deployment
         uses: actions/deploy-pages@v4
diff --git a/.gitignore b/.gitignore
@@ -1,3 +1,5 @@
+data/
+
 target/
 dbt_packages/
 logs/

diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml
@@ -7,6 +7,7 @@ profile: 'integration_tests'
 test-paths: ["tests"]
 seed-paths: ["seeds"]
 macro-paths: ["macros"]
+model-paths: ["models"]
 
 target-path: "target"
 clean-targets:

diff --git a/integration_tests/models/sp500_agg.sql b/integration_tests/models/sp500_agg.sql
@@ -0,0 +1,17 @@
+{{
+    config(alias='sp500_agg_alias')
+}}
+WITH
+formatted as (
+    SELECT
+        date_part('dayofweek', day) as day_of_week,
+        high,
+        low
+    FROM {{ ref('sp500_daily') }}
+)
+SELECT
+    day_of_week,
+    MAX(high) as high,
+    MIN(low) as low
+FROM formatted
+GROUP BY day_of_week
diff --git a/integration_tests/package-lock.yml b/integration_tests/package-lock.yml
@@ -0,0 +1,9 @@
+packages:
+- local: ../
+- package: dbt-labs/dbt_utils
+  version: 1.1.1
+- package: calogica/dbt_expectations
+  version: 0.10.1
+- package: calogica/dbt_date
+  version: 0.10.0
+sha1_hash: 4d5769928d899374729d323ba119162d793c03fd
diff --git a/integration_tests/tests/helpers/get_relation_name_1.sql b/integration_tests/tests/helpers/get_relation_name_1.sql
@@ -0,0 +1,4 @@
+
+{% set tablename = testgen.get_relation_name('sp500_agg') %}
+
+{{ assert_equal(tablename, 'sp500_agg') }}
diff --git a/integration_tests/tests/helpers/get_relation_name_2.sql b/integration_tests/tests/helpers/get_relation_name_2.sql
@@ -0,0 +1,4 @@
+
+{% set tablename = testgen.get_relation_name(ref('sp500_agg')) %}
+
+{{ assert_equal(tablename, 'sp500_agg_alias') }}
diff --git a/integration_tests/tests/test_test_selection/test_selection_sp500_agg.sql b/integration_tests/tests/test_test_selection/test_selection_sp500_agg.sql
@@ -0,0 +1,63 @@
+-- depends_on: {{ ref('sp500_agg') }}
+
+{{ config(
+    tags="dataset-sp500_daily"
+) }}
+
+{% set actual_yaml = testgen.to_yaml(
+        testgen.get_test_suggestions(
+            'sp500_agg',
+            return_object=true
+        )
+    )
+%}
+
+{% set expected_yaml %}
+models:
+- name: sp500_agg
+  columns:
+  - name: day_of_week
+    tests:
+    - unique
+    - not_null
+    - accepted_values:
+        values:
+        - '1'
+        - '2'
+        - '3'
+        - '4'
+        - '5'
+    - dbt_utils.accepted_range:
+        min_value: 1
+        max_value: 5
+  - name: high
+    tests:
+    - unique
+    - not_null
+    - accepted_values:
+        values:
+        - '4749.52'
+        - '4784.72'
+        - '4785.39'
+        - '4788.43'
+        - '4793.3'
+    - dbt_utils.accepted_range:
+        min_value: 4749.52
+        max_value: 4793.3
+  - name: low
+    tests:
+    - unique
+    - not_null
+    - accepted_values:
+        values:
+        - '4546.5'
+        - '4546.72'
+        - '4551.68'
+        - '4565.22'
+        - '4574.06'
+    - dbt_utils.accepted_range:
+        min_value: 4546.5
+        max_value: 4574.06
+{% endset %}
+
+{{ assert_equal (actual_yaml | trim, expected_yaml | trim) }}
diff --git a/macros/helpers/relations.sql b/macros/helpers/relations.sql
@@ -0,0 +1,26 @@
+
+{% macro get_relation_name(relation_name) %}
+    -- {% print('here1') %}
+
+    -- {% print(relation_name.identifier) %}
+
+    {% if relation_name.identifier | trim == '' %}
+        {{ return(relation_name) }}
+    {% endif %}
+
+    {{ return(relation_name.identifier) }}
+{% endmacro %}
+
+
+{% macro get_relation(relation_name) %}
+    -- {% print('here1') %}
+
+    -- {% print(relation_name.identifier) %}
+
+    {% if relation_name.identifier | trim == '' %}
+        {{ return(ref(relation_name)) }}
+    {% endif %}
+
+    {{ return(relation_name) }}
+{% endmacro %}
+
diff --git a/macros/test_aggregation/get_test_suggestions.sql b/macros/test_aggregation/get_test_suggestions.sql
@@ -1,6 +1,6 @@
 
 {% macro get_test_suggestions(
-        table_relation,
+        relation_name,
         sample = false,
         limit = 10000,
         resource_type = "models",
@@ -20,7 +20,7 @@
     {% if execute %}
         {% if "uniqueness" in tests %}
             {% set dbt_config = testgen.get_uniqueness_test_suggestions(
-                table_relation=table_relation,
+                relation_name=relation_name,
                 sample=sample,
                 limit=limit,
                 resource_type=resource_type,
@@ -34,7 +34,7 @@
 
         {% if "accepted_values" in tests %}
             {% set dbt_config = testgen.get_accepted_values_test_suggestions(
-                table_relation=table_relation,
+                relation_name=relation_name,
                 sample=sample,
                 limit=limit,
                 resource_type=resource_type,
@@ -48,7 +48,7 @@
 
         {% if "range" in tests %}
             {% set dbt_config = testgen.get_range_test_suggestions(
-                table_relation=table_relation,
+                relation_name=relation_name,
                 sample=sample,
                 limit=limit,
                 resource_type=resource_type,
@@ -62,7 +62,7 @@
 
         {% if "string_length" in tests %}
             {% set dbt_config = testgen.get_string_length_test_suggestions(
-                table_relation=table_relation,
+                relation_name=relation_name,
                 sample=sample,
                 limit=limit,
                 resource_type=resource_type,
@@ -76,7 +76,7 @@
 
         {% if "recency" in tests %}
             {% set dbt_config = testgen.get_recency_test_suggestions(
-                table_relation=table_relation,
+                relation_name=relation_name,
                 sample=sample,
                 limit=limit,
                 resource_type=resource_type,

diff --git a/macros/test_generation/get_accepted_values_test_suggestions.sql b/macros/test_generation/get_accepted_values_test_suggestions.sql
@@ -20,7 +20,7 @@
 
 
 {% macro get_accepted_values_test_suggestions(
-        table_relation,
+        relation_name,
         sample = false,
         limit = None,
         resource_type = "models",
@@ -34,7 +34,7 @@
     {% if execute %}
         {{ return(
             adapter.dispatch('get_accepted_values_test_suggestions', 'testgen')(
-                table_relation, 
+                relation_name, 
                 sample, 
                 limit, 
                 resource_type,
@@ -51,7 +51,7 @@
 
 
 {% macro default__get_accepted_values_test_suggestions(
-        table_relation,
+        relation_name,
         sample = false,
         limit = None,
         resource_type = "models",
@@ -62,7 +62,9 @@
         dbt_config = None
     ) 
 %}
-    {% set columns = adapter.get_columns_in_relation(table_relation) %}
+    {% set relation_name = testgen.get_relation_name(relation_name) %}
+    {% set relation = testgen.get_relation(relation_name) %}
+    {% set columns = adapter.get_columns_in_relation(relation) %}
     {% set columns = testgen.exclude_column_types(columns, exclude_types) %}
     {% set columns = testgen.exclude_column_names(columns, exclude_cols) %}
 
@@ -103,7 +105,7 @@
 
     {% set count_distinct_sql %}
         WITH base AS (
-            SELECT * FROM {{ table_relation }}
+            SELECT * FROM {{ relation }}
             {{ limit_stmt }}
         )
         SELECT * FROM (
@@ -139,7 +141,7 @@
         {% do column_tests.append(col_config) %}
     {% endfor %}
 
-    {% set new_dbt_config = {resource_type: [{"name": table_relation.identifier, "columns": column_tests}]} %}
+    {% set new_dbt_config = {resource_type: [{"name": testgen.get_relation_name(relation_name), "columns": column_tests}]} %}
 
     {# {{ print(new_dbt_config) }} #}
 

diff --git a/macros/test_generation/get_range_test_suggestions.sql b/macros/test_generation/get_range_test_suggestions.sql
@@ -1,6 +1,6 @@
 
 {% macro get_range_test_suggestions(
-        table_relation,
+        relation_name,
         sample = false,
         limit = None,
         resource_type = "models",
@@ -12,13 +12,13 @@
     ) %}
     {# Run macro for the specific target DB #}
     {% if execute %}
-        {{ return(adapter.dispatch('get_range_test_suggestions', 'testgen')(table_relation, sample, limit, resource_type, column_config, exclude_types, exclude_cols, stddevs, dbt_config, **kwargs)) }}
+        {{ return(adapter.dispatch('get_range_test_suggestions', 'testgen')(relation_name, sample, limit, resource_type, column_config, exclude_types, exclude_cols, stddevs, dbt_config, **kwargs)) }}
     {% endif%}
 {%- endmacro %}
 
 
 {% macro default__get_range_test_suggestions(
-        table_relation,
+        relation_name,
         sample = false,
         limit = None,
         resource_type = "models",
@@ -29,7 +29,9 @@
         dbt_config = None
     ) 
 %}
-    {% set columns = adapter.get_columns_in_relation(table_relation) %}
+    {% set relation_name = testgen.get_relation_name(relation_name) %}
+    {% set relation = testgen.get_relation(relation_name) %}
+    {% set columns = adapter.get_columns_in_relation(relation) %}
     {% set columns = testgen.exclude_column_types(columns, exclude_types) %}
     {% set columns = testgen.exclude_column_names(columns, exclude_cols) %}
 
@@ -69,7 +71,7 @@
 
     {% set min_max_sql %}
         WITH base AS (
-            SELECT * FROM {{ table_relation }}
+            SELECT * FROM {{ relation }}
             {{ limit_stmt }}
         )
         SELECT * FROM (
@@ -105,7 +107,7 @@
         {% do column_tests.append(col_config) %}
     {% endfor %}
 
-    {% set model = {"name": table_relation.identifier,  "columns": column_tests} %}
+    {% set model = {"name": testgen.get_relation_name(relation_name),  "columns": column_tests} %}
 
     {% set new_dbt_config = {resource_type: [model]} %}
 

diff --git a/macros/test_generation/get_recency_test_suggestions.sql b/macros/test_generation/get_recency_test_suggestions.sql
@@ -1,6 +1,6 @@
 
 {% macro get_recency_test_suggestions(
-        table_relation,
+        relation_name,
         sample = false,
         limit = None,
         resource_type = "models",
@@ -14,7 +14,7 @@
     {% if execute %}
         {{ return(
             adapter.dispatch('get_recency_test_suggestions', 'testgen')(
-                table_relation, 
+                relation_name, 
                 sample, 
                 limit, 
                 resource_type,
@@ -31,7 +31,7 @@
 
 
 {% macro default__get_recency_test_suggestions(
-        table_relation,
+        relation_name,
         sample = false,
         limit = None,
         resource_type = "models",
@@ -42,7 +42,9 @@
         dbt_config = None
     ) 
 %}
-    {% set columns = adapter.get_columns_in_relation(table_relation) %}
+    {% set relation_name = testgen.get_relation_name(relation_name) %}
+    {% set relation = testgen.get_relation(relation_name) %}
+    {% set columns = adapter.get_columns_in_relation(relation) %}
     {% set columns = testgen.exclude_column_types(columns, exclude_types) %}
     {% set columns = testgen.exclude_column_names(columns, exclude_cols) %}
 
@@ -87,7 +89,7 @@
 
     {% set timestep_sql %}
     WITH base AS (
-            SELECT * FROM {{ table_relation }}
+            SELECT * FROM {{ relation }}
             {{ limit_stmt }}
         )
     {{ timestep_exprs | join("\nUNION ALL\n") }}
@@ -128,7 +130,7 @@
 
 
 
-    {% set model = {"name": table_relation.identifier} %}
+    {% set model = {"name": testgen.get_relation_name(relation_name)} %}
     {% if table_tests != [] %}
         {% do model.update({"tests": table_tests}) %} 
     {% endif %}
-Original file line number
+Diff line change
@@ -1,3 +1,5 @@
+    data/
     target/
     dbt_packages/
     logs/
@@ Expand Down @@
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,4 @@

		{% set tablename = testgen.get_relation_name('sp500_agg') %}

		{{ assert_equal(tablename, 'sp500_agg') }}
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,4 @@

		{% set tablename = testgen.get_relation_name(ref('sp500_agg')) %}

		{{ assert_equal(tablename, 'sp500_agg_alias') }}