From 87dddd961b60a15475b8e8f8869e4b84aaec91cc Mon Sep 17 00:00:00 2001 From: kevin Date: Wed, 3 Jan 2024 22:41:27 -0600 Subject: [PATCH] fixed tests --- .github/workflows/dbt_test.yml | 36 +++++++++++++------ integration_tests/seeds/sp500_daily.csv | 2 +- integration_tests/seeds/sp500_monthly.csv | 2 +- .../recency_sp500_daily.sql | 2 +- .../recency_sp500_monthly.sql | 2 +- .../test_selection_sp500_daily.sql | 4 +-- .../get_recency_test_suggestions.sql | 2 +- 7 files changed, 32 insertions(+), 18 deletions(-) diff --git a/.github/workflows/dbt_test.yml b/.github/workflows/dbt_test.yml index d111097..22f3411 100644 --- a/.github/workflows/dbt_test.yml +++ b/.github/workflows/dbt_test.yml @@ -37,12 +37,15 @@ jobs: - name: load test data run: dbt seed --target ${DBT_TARGET} + + - name: dbt test macros + run: dbt test --target ${DBT_TARGET} - name: Create seed tests run: bash tests/create_seeds_tests_yml.sh - - name: dbt test - run: dbt test --target ${DBT_TARGET} --exclude "*recency*" + - name: dbt test seeds + run: dbt test --target ${DBT_TARGET} --select config.materialized:seed --exclude "dbt_utils_recency*" test-postgres: name: Test on Postgres @@ -71,11 +74,14 @@ jobs: - name: load test data run: dbt seed --target ${DBT_TARGET} + - name: dbt test macros + run: dbt test --target ${DBT_TARGET} + - name: Create seed tests run: bash tests/create_seeds_tests_yml.sh - - name: dbt test - run: dbt test --target ${DBT_TARGET} --exclude "*recency*" + - name: dbt test seeds + run: dbt test --target ${DBT_TARGET} --select config.materialized:seed --exclude "dbt_utils_recency*" test-snowflake: name: Test on Snowflake @@ -99,11 +105,14 @@ jobs: - name: load test data run: dbt seed --target ${DBT_TARGET} + - name: dbt test macros + run: dbt test --target ${DBT_TARGET} + - name: Create seed tests run: bash tests/create_seeds_tests_yml.sh - - name: dbt test - run: dbt test --target ${DBT_TARGET} --exclude "*recency*" + - name: dbt test seeds + run: dbt test --target ${DBT_TARGET} --select config.materialized:seed --exclude "dbt_utils_recency*" test-redshift: name: Test on RedShift @@ -127,11 +136,14 @@ jobs: - name: load test data run: dbt seed --target ${DBT_TARGET} + - name: dbt test macros + run: dbt test --target ${DBT_TARGET} + - name: Create seed tests run: bash tests/create_seeds_tests_yml.sh - - name: dbt test - run: dbt test --target ${DBT_TARGET} --exclude "*recency*" + - name: dbt test seeds + run: dbt test --target ${DBT_TARGET} --select config.materialized:seed --exclude "dbt_utils_recency*" test-bigquery: name: Test on BigQuery @@ -164,14 +176,16 @@ jobs: - name: load test data run: dbt seed --target ${DBT_TARGET} -s users + - name: dbt test macros + run: dbt test --target ${DBT_TARGET} + - name: Create seed tests run: | dbt compile --target ${DBT_TARGET} -q \ --inline "{{ testgen.get_test_suggestions(ref('users'), resource_type='seeds', column_config={'quote': true, 'tags': ['dataset-users']} ) }}" \ > seeds/users_test_suggestions.yml - - name: dbt test - run: dbt test --target ${DBT_TARGET} -s tag:dataset-users - + - name: dbt test seeds + run: dbt test --target ${DBT_TARGET} --select config.materialized:seed --exclude "dbt_utils_recency*" \ No newline at end of file diff --git a/integration_tests/seeds/sp500_daily.csv b/integration_tests/seeds/sp500_daily.csv index c142d59..7c91df4 100644 --- a/integration_tests/seeds/sp500_daily.csv +++ b/integration_tests/seeds/sp500_daily.csv @@ -1,4 +1,4 @@ -date,close,open,high,low +day,close,open,high,low 2024-01-02,4742.83,4745.20,4754.33,4722.67 2023-12-29,4769.83,4782.88,4788.43,4751.99 2023-12-28,4783.35,4786.44,4793.30,4780.98 diff --git a/integration_tests/seeds/sp500_monthly.csv b/integration_tests/seeds/sp500_monthly.csv index 4f3b75c..1cd1785 100644 --- a/integration_tests/seeds/sp500_monthly.csv +++ b/integration_tests/seeds/sp500_monthly.csv @@ -1,4 +1,4 @@ -date,sp500,dividend,earnings,consumer_price_index,long_interest_rate,real_price,real_dividend,real_earnings,pe10 +month,sp500,dividend,earnings,consumer_price_index,long_interest_rate,real_price,real_dividend,real_earnings,pe10 1871-01-01,4.44,0.26,0.4,12.46,5.32,89.0,5.21,8.02, 1871-02-01,4.5,0.26,0.4,12.84,5.32,87.53,5.06,7.78, 1871-03-01,4.61,0.26,0.4,13.03,5.33,88.36,4.98,7.67, diff --git a/integration_tests/tests/generate_recency_tests/recency_sp500_daily.sql b/integration_tests/tests/generate_recency_tests/recency_sp500_daily.sql index de6c88b..372f8cd 100644 --- a/integration_tests/tests/generate_recency_tests/recency_sp500_daily.sql +++ b/integration_tests/tests/generate_recency_tests/recency_sp500_daily.sql @@ -13,7 +13,7 @@ models: - name: sp500_daily tests: - dbt_utils.recency: - field: date + field: day datepart: day interval: 2 {% endset %} diff --git a/integration_tests/tests/generate_recency_tests/recency_sp500_monthly.sql b/integration_tests/tests/generate_recency_tests/recency_sp500_monthly.sql index 5361890..f95babd 100644 --- a/integration_tests/tests/generate_recency_tests/recency_sp500_monthly.sql +++ b/integration_tests/tests/generate_recency_tests/recency_sp500_monthly.sql @@ -13,7 +13,7 @@ models: - name: sp500_monthly tests: - dbt_utils.recency: - field: Date + field: month datepart: day interval: 31 {% endset %} diff --git a/integration_tests/tests/test_test_selection/test_selection_sp500_daily.sql b/integration_tests/tests/test_test_selection/test_selection_sp500_daily.sql index 09b3459..151186f 100644 --- a/integration_tests/tests/test_test_selection/test_selection_sp500_daily.sql +++ b/integration_tests/tests/test_test_selection/test_selection_sp500_daily.sql @@ -15,11 +15,11 @@ models: - name: sp500_daily tests: - dbt_utils.recency: - field: date + field: day datepart: day interval: 2 columns: - - name: date + - name: day tests: - unique - not_null diff --git a/macros/test_generation/get_recency_test_suggestions.sql b/macros/test_generation/get_recency_test_suggestions.sql index 0e874db..d6ce255 100644 --- a/macros/test_generation/get_recency_test_suggestions.sql +++ b/macros/test_generation/get_recency_test_suggestions.sql @@ -86,7 +86,7 @@ {{ loop.index }} AS ordering FROM ( SELECT - {{ dbt.datediff("LAG(date, 1) OVER(ORDER BY " ~ adapter.quote(col.column) ~ ")", adapter.quote(col.column), "minute") }} AS minutes_diff + {{ dbt.datediff("LAG(" ~ adapter.quote(col.column) ~ ", 1) OVER(ORDER BY " ~ adapter.quote(col.column) ~ ")", adapter.quote(col.column), "minute") }} AS minutes_diff FROM base ) t2 WHERE minutes_diff <> 0