Feature/contact list api updates

CHANGELOG.md

@@ -1,3 +1,13 @@
+# dbt_hubspot v0.22.0
+## Breaking Changes
+- TBD
+
+## Documentation
+- TBD
+
+## Under the Hood
+- TBD
+
 # dbt_hubspot v0.21.0
 This release includes the following updates:
 

README.md

@@ -79,7 +79,7 @@ Include the following hubspot package version in your `packages.yml` file:
 ```yaml
 packages:
   - package: fivetran/hubspot
-    version: [">=0.21.0", "<0.22.0"] # we recommend using ranges to capture non-breaking changes automatically
+    version: [">=0.22.0", "<0.23.0"] # we recommend using ranges to capture non-breaking changes automatically
 
 ```
 Do **NOT** include the `hubspot_source` package in this file. The transformation package itself has a dependency on it and will install the source package as well.
@@ -321,7 +321,7 @@ This dbt package is dependent on the following dbt packages. These dependencies
 ```yml
 packages:
     - package: fivetran/hubspot_source
-      version: [">=0.17.0", "<0.18.0"]
+      version: [">=0.18.0", "<0.19.0"]
 
     - package: fivetran/fivetran_utils
       version: [">=0.4.0", "<0.5.0"]

dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'hubspot'
-version: '0.21.0'
+version: '0.22.0'
 
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]

integration_tests/dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'hubspot_integration_tests'
-version: '0.21.0'
+version: '0.22.0'
 
 profile: 'integration_tests'
 config-version: 2
@@ -10,11 +10,18 @@ models:
 
 vars:
   hubspot_schema: hubspot_integration_tests_63
+  consistency_test_contact_list_exclude_fields: ## For hubspot__contact_lists consistency test. Can be removed after this coming release.
+    - created_by_id
+    - object_type_id
+    - processing_status
+    - processing_type
+    - list_version
+    - filters_updated_at
+
   # hubspot__pass_through_all_columns: true
   # hubspot__company_pass_through_columns:
     # - name: "property_hs_all-funky-a9384-syntax"
     #   alias: "funky_field"
-
   # hubspot_service_enabled: true # enable when generating docs
   # hubspot_deal_enabled: true # enable when generating docs
   # hubspot_contact_enabled: true # enable when generating docs
@@ -200,11 +207,6 @@ seeds:
       +enabled: "{{ true if target.type == 'postgres' else false }}"
     email_event_dropped_data:
       +enabled: "{{ true if target.type not in ('snowflake', 'postgres') else false }}"
-    contact_list_data:
-      +enabled: "{{ true if target.type != 'postgres' else false }}"
-    contact_list_data_postgres:
-      +alias: contact_list_data
-      +enabled: "{{ true if target.type == 'postgres' else false }}"
     email_event_data: 
       +column_types:
         _fivetran_synced: timestamp

integration_tests/tests/consistency/row_comparisons/consistency_contact_list.sql

@@ -0,0 +1,54 @@
+{{ config(
+    tags="fivetran_validations",
+    enabled=var('fivetran_validation_tests_enabled', false) and var('hubspot_contact_list_enabled', true)
+) }}
+
+-- this test ensures the contact_list end model matches the prior version
+with prod as (
+    select 
+        {{ dbt_utils.star(
+            from=ref('hubspot__contact_lists'), 
+            except=var('consistency_test_contact_list_exclude_fields', [])) 
+        }}
+    from {{ target.schema }}_hubspot_prod.hubspot__contact_lists
+),
+
+dev as (
+    select 
+        {{ dbt_utils.star(
+            from=ref('hubspot__contact_lists'), 
+            except=var('consistency_test_contact_list_exclude_fields', [])) 
+        }}
+    from {{ target.schema }}_hubspot_dev.hubspot__contact_lists
+),
+
+prod_not_in_dev as (
+    -- rows from prod not found in dev
+    select * from prod
+    except distinct
+    select * from dev
+),
+
+dev_not_in_prod as (
+    -- rows from dev not found in prod
+    select * from dev
+    except distinct
+    select * from prod
+),
+
+final as (
+    select
+        *,
+        'from prod' as source
+    from prod_not_in_dev
+
+    union all -- union since we only care if rows are produced
+
+    select
+        *,
+        'from dev' as source
+    from dev_not_in_prod
+)
+
+select *
+from final

integration_tests/tests/consistency/row_comparisons/consistency_daily_ticket_history.sql

@@ -1,6 +1,6 @@
 {{ config(
     tags="fivetran_validations",
-    enabled=var('fivetran_validation_tests_enabled', false)
+    enabled=var('fivetran_validation_tests_enabled', false) and var('hubspot_service_enabled', false)
 ) }}
 
 -- this test ensures the daily_activity end model matches the prior version

integration_tests/tests/consistency/row_counts/consistency_contact_lists_count.sql

@@ -0,0 +1,21 @@
+{{ config(
+    tags="fivetran_validations",
+    enabled=var('fivetran_validation_tests_enabled', false) and var('hubspot_contact_list_enabled', true)
+) }}
+
+-- this test is to make sure the rows counts are the same between versions
+with prod as (
+    select count(*) as prod_rows
+    from {{ target.schema }}_hubspot_prod.hubspot__contact_lists
+),
+
+dev as (
+    select count(*) as dev_rows
+    from {{ target.schema }}_hubspot_dev.hubspot__contact_lists
+)
+
+-- test will return values and fail if the row counts don't match
+select *
+from prod
+cross join dev
+where prod.prod_rows != dev.dev_rows

integration_tests/tests/consistency/row_counts/consistency_daily_ticket_history_count.sql

@@ -1,6 +1,6 @@
 {{ config(
     tags="fivetran_validations",
-    enabled=var('fivetran_validation_tests_enabled', false)
+    enabled=var('fivetran_validation_tests_enabled', false) and var('hubspot_service_enabled', false)
 ) }}
 
 -- this test is to make sure the rows counts are the same between versions

integration_tests/tests/integrity/integrity_contact_lists_count.sql

@@ -0,0 +1,20 @@
+{{ config(
+    tags="fivetran_validations",
+    enabled=var('fivetran_validation_tests_enabled', false) and var('hubspot_contact_list_enabled', true)
+) }}
+
+-- This test will ensure the total count of records from the contact_list staging model matches the end model.
+with end_model as (
+    select count(*) as end_model_rows
+    from {{ ref('hubspot__contact_lists') }}
+),
+
+staging_model as (
+    select count(*) as staging_model_rows
+    from {{ ref('stg_hubspot__contact_list') }}
+)
+
+select *
+from end_model
+cross join staging_model
+where end_model_rows != staging_model_rows