Add original_reason_entitlement_code to Eligibility (#55)

* Add new field * Replace custom string macro * Add underscore to schema name for models created by the connector * Add claims enabled var and bump project version * Update package version to latest minor release * Fix data types causing errors * Materialize final tables to prevent issues coming up during CI testing * Add tuva_schema_prefix variable to Github actions to help prevent overlap with other runners from other repos
tuva-health · Sep 27, 2023 · 419dbb7 · 419dbb7
1 parent 594c9ca
commit 419dbb7
Show file tree

Hide file tree

Showing 17 changed files with 396 additions and 433 deletions.
diff --git a/.github/workflows/biweekly_dbt_v1.3.0_bigquery_build_full_refresh.yml b/.github/workflows/biweekly_dbt_v1.3.0_bigquery_build_full_refresh.yml
@@ -33,7 +33,7 @@ jobs:
       - name: dbt-build
         uses: mwhitaker/[email protected]
         with:
-          dbt_command: 'dbt build --full-refresh --profiles-dir ./integration_tests/profiles/bigquery  --vars {"tuva_database":"dev-ci-testing","input_database":"dev-ci-testing","input_schema":"cclf"}'
+          dbt_command: 'dbt build --full-refresh --profiles-dir ./integration_tests/profiles/bigquery  --vars {"tuva_database":"dev-ci-testing","input_database":"dev-ci-testing","input_schema":"cclf","tuva_schema_prefix":"cclf"}'
 
 
       - name: Get the result

diff --git a/.github/workflows/biweekly_dbt_v1.3.0_redshift_build_full_refresh.yml b/.github/workflows/biweekly_dbt_v1.3.0_redshift_build_full_refresh.yml
@@ -33,7 +33,7 @@ jobs:
       - name: dbt-build
         uses: mwhitaker/[email protected]
         with:
-          dbt_command: 'dbt build --full-refresh --profiles-dir ./integration_tests/profiles/redshift --vars {"tuva_database":"dev_ci_testing","input_database":"dev_ci_testing","input_schema":"cclf"}'
+          dbt_command: 'dbt build --full-refresh --profiles-dir ./integration_tests/profiles/redshift --vars {"tuva_database":"dev_ci_testing","input_database":"dev_ci_testing","input_schema":"cclf","tuva_schema_prefix":"cclf"}'
 
       - name: Get the result
         if: ${{ always() }}

diff --git a/.github/workflows/biweekly_dbt_v1.3.0_snowflake_build_full_refresh.yml b/.github/workflows/biweekly_dbt_v1.3.0_snowflake_build_full_refresh.yml
@@ -35,7 +35,7 @@ jobs:
       - name: dbt-build
         uses: mwhitaker/[email protected]
         with:
-          dbt_command: 'dbt build --full-refresh --profiles-dir ./integration_tests/profiles/snowflake --vars {"tuva_database":"dev_ci_testing","input_database":"dev_ci_testing","input_schema":"cclf"}'
+          dbt_command: 'dbt build --full-refresh --profiles-dir ./integration_tests/profiles/snowflake --vars {"tuva_database":"dev_ci_testing","input_database":"dev_ci_testing","input_schema":"cclf","tuva_schema_prefix":"cclf"}'
 
       - name: Get the result
         if: ${{ always() }}

diff --git a/.github/workflows/dbt_v1.4.6_bigquery_build_full_refresh.yml b/.github/workflows/dbt_v1.4.6_bigquery_build_full_refresh.yml
@@ -30,7 +30,7 @@ jobs:
       - name: dbt-build
         uses: mwhitaker/[email protected]
         with:
-          dbt_command: 'dbt build --full-refresh --profiles-dir ./integration_tests/profiles/bigquery  --vars {"tuva_database":"dev-ci-testing","input_database":"dev-ci-testing","input_schema":"cclf"}'
+          dbt_command: 'dbt build --full-refresh --profiles-dir ./integration_tests/profiles/bigquery  --vars {"tuva_database":"dev-ci-testing","input_database":"dev-ci-testing","input_schema":"cclf","tuva_schema_prefix":"cclf"}'
 
 
       - name: Get the result

diff --git a/.github/workflows/dbt_v1.4.6_redshift_build_full_refresh.yml b/.github/workflows/dbt_v1.4.6_redshift_build_full_refresh.yml
@@ -31,7 +31,7 @@ jobs:
       - name: dbt-build
         uses: mwhitaker/[email protected]
         with:
-          dbt_command: 'dbt build --full-refresh --profiles-dir ./integration_tests/profiles/redshift --vars {"tuva_database":"dev_ci_testing","input_database":"dev_ci_testing","input_schema":"cclf"}'
+          dbt_command: 'dbt build --full-refresh --profiles-dir ./integration_tests/profiles/redshift --vars {"tuva_database":"dev_ci_testing","input_database":"dev_ci_testing","input_schema":"cclf","tuva_schema_prefix":"cclf"}'
 
       - name: Get the result
         if: ${{ always() }}

diff --git a/.github/workflows/dbt_v1.4.6_snowflake_build_full_refresh.yml b/.github/workflows/dbt_v1.4.6_snowflake_build_full_refresh.yml
@@ -35,7 +35,7 @@ jobs:
       - name: dbt-build
         uses: mwhitaker/[email protected]
         with:
-          dbt_command: 'dbt build --full-refresh --profiles-dir ./integration_tests/profiles/snowflake --vars {"tuva_database":"dev_ci_testing","input_database":"dev_ci_testing","input_schema":"cclf"}'
+          dbt_command: 'dbt build --full-refresh --profiles-dir ./integration_tests/profiles/snowflake --vars {"tuva_database":"dev_ci_testing","input_database":"dev_ci_testing","input_schema":"cclf","tuva_schema_prefix":"cclf"}'
 
 
       - name: Get the result

diff --git a/.github/workflows/outside_contributor_dbt_v1.4.6_all_build_full_refresh.yml b/.github/workflows/outside_contributor_dbt_v1.4.6_all_build_full_refresh.yml
@@ -47,7 +47,7 @@ jobs:
       - name: dbt-build
         uses: mwhitaker/[email protected]
         with:
-          dbt_command: 'dbt build --full-refresh --profiles-dir ./integration_tests/profiles/snowflake --vars {"tuva_database":"dev_ci_testing","input_database":"dev_ci_testing","input_schema":"cclf"}'
+          dbt_command: 'dbt build --full-refresh --profiles-dir ./integration_tests/profiles/snowflake --vars {"tuva_database":"dev_ci_testing","input_database":"dev_ci_testing","input_schema":"cclf","tuva_schema_prefix":"cclf"}'
 
       - name: Get the result
         if: ${{ always() }}
@@ -76,7 +76,7 @@ jobs:
       - name: dbt-build
         uses: mwhitaker/[email protected]
         with:
-          dbt_command: 'dbt build --full-refresh --profiles-dir ./integration_tests/profiles/redshift --vars {"tuva_database":"dev_ci_testing","input_database":"dev_ci_testing","input_schema":"cclf"}'
+          dbt_command: 'dbt build --full-refresh --profiles-dir ./integration_tests/profiles/redshift --vars {"tuva_database":"dev_ci_testing","input_database":"dev_ci_testing","input_schema":"cclf","tuva_schema_prefix":"cclf"}'
 
       - name: Get the result
         if: ${{ always() }}
@@ -106,7 +106,7 @@ jobs:
       - name: dbt-build
         uses: mwhitaker/[email protected]
         with:
-          dbt_command: 'dbt build --full-refresh --profiles-dir ./integration_tests/profiles/bigquery  --vars {"tuva_database":"dev-ci-testing","input_database":"dev-ci-testing","input_schema":"cclf"}'
+          dbt_command: 'dbt build --full-refresh --profiles-dir ./integration_tests/profiles/bigquery  --vars {"tuva_database":"dev-ci-testing","input_database":"dev-ci-testing","input_schema":"cclf","tuva_schema_prefix":"cclf"}'
 
       - name: Get the result
         if: ${{ always() }}

diff --git a/dbt_project.yml b/dbt_project.yml
@@ -1,14 +1,16 @@
 name: 'medicare_cclf_connector'
-version: '0.1.1'
+version: '0.1.2'
 config-version: 2
 
 profile: default
 
 vars:
 
-## Set the database and schema where raw CCLF data is stored
+  ## Set the database and schema where raw CCLF data is stored
   input_database: default
-  input_schema: default    
+  input_schema: default
+
+  claims_enabled: true
 
 model-paths: ["models"]
 analysis-paths: ["analyses"]

diff --git a/macros/cast_string_or_varchar.sql b/macros/cast_string_or_varchar.sql
diff --git a/models/_models.yml b/models/_models.yml
@@ -1,79 +1,77 @@
 version: 2
 
 models:
-
-### staging
-  - name: base_parta_diagnosis_code
-    description: Base model used as ref for pivot transformation
+### final
+  - name: eligibility
+    description: >
+      Member enrollment and demographic detail, converts member months to 
+      enrollment spans.
     config:
       schema: |
-        {%- if var('medicare_cclf_connector_schema',None) != None -%}{{ var('medicare_cclf_connector_schema') }}{%- elif var('tuva_schema_prefix',None) != None -%}{{var('tuva_schema_prefix')}}_tuva_claims{% else %}tuva_claims{%- endif -%}
+        {%- if var('tuva_schema_prefix',None) != None -%}_{{var('tuva_schema_prefix')}}_tuva_claims{% else %}_tuva_claims{%- endif -%}
+      materialized: table
 
-
-  - name: base_parta_procedure_code
-    description: Base model used as ref for pivot transformation
+  - name: medical_claim
+    description: Claim line level medical data.
     config:
       schema: |
-        {%- if var('medicare_cclf_connector_schema',None) != None -%}{{ var('medicare_cclf_connector_schema') }}{%- elif var('tuva_schema_prefix',None) != None -%}{{var('tuva_schema_prefix')}}_tuva_claims{% else %}tuva_claims{%- endif -%}
+        {%- if var('tuva_schema_prefix',None) != None -%}_{{var('tuva_schema_prefix')}}_tuva_claims{% else %}_tuva_claims{%- endif -%}
+      materialized: table
 
+  - name: pharmacy_claim
+    description: Claim line level pharmacy data.
+    config:
+      schema: |
+        {%- if var('tuva_schema_prefix',None) != None -%}_{{var('tuva_schema_prefix')}}_tuva_claims{% else %}_tuva_claims{%- endif -%}
+      materialized: table
 
 ### intermediate
   - name: diagnosis_pivot
-    description: Pivot of diagnosis from long (1 row/diagnosis) to wide (multiple diagnosis per row)
+    description: >
+      Pivot of diagnosis from long (1 row/diagnosis) to wide (multiple diagnosis
+      per row).
     config:
       schema: |
-        {%- if var('medicare_cclf_connector_schema',None) != None -%}{{ var('medicare_cclf_connector_schema') }}{%- elif var('tuva_schema_prefix',None) != None -%}{{var('tuva_schema_prefix')}}_tuva_claims{% else %}tuva_claims{%- endif -%}
-
+        {%- if var('tuva_schema_prefix',None) != None -%}_{{var('tuva_schema_prefix')}}_tuva_claims{% else %}_tuva_claims{%- endif -%}
 
   - name: dme_claims
-    description: Mapping of durable medical equipment claims to Tuva claims data model
+    description: >
+      Mapping of durable medical equipment claims to Tuva claims data model.
     config:
       schema: |
-        {%- if var('medicare_cclf_connector_schema',None) != None -%}{{ var('medicare_cclf_connector_schema') }}{%- elif var('tuva_schema_prefix',None) != None -%}{{var('tuva_schema_prefix')}}_tuva_claims{% else %}tuva_claims{%- endif -%}
-
+        {%- if var('tuva_schema_prefix',None) != None -%}_{{var('tuva_schema_prefix')}}_tuva_claims{% else %}_tuva_claims{%- endif -%}
 
   - name: institutional_claims
     description: >
-      Mapping of institution claims to Tuva claims data model
+      Mapping of institution claims to Tuva claims data model.
     config:
       schema: |
-        {%- if var('medicare_cclf_connector_schema',None) != None -%}{{ var('medicare_cclf_connector_schema') }}{%- elif var('tuva_schema_prefix',None) != None -%}{{var('tuva_schema_prefix')}}_tuva_claims{% else %}tuva_claims{%- endif -%}
-
-
+        {%- if var('tuva_schema_prefix',None) != None -%}_{{var('tuva_schema_prefix')}}_tuva_claims{% else %}_tuva_claims{%- endif -%}
 
   - name: physician_claims
     description: >
-      Mapping of professional claims to Tuva claims data model
+      Mapping of professional claims to Tuva claims data model.
     config:
       schema: |
-        {%- if var('medicare_cclf_connector_schema',None) != None -%}{{ var('medicare_cclf_connector_schema') }}{%- elif var('tuva_schema_prefix',None) != None -%}{{var('tuva_schema_prefix')}}_tuva_claims{% else %}tuva_claims{%- endif -%}
-
+        {%- if var('tuva_schema_prefix',None) != None -%}_{{var('tuva_schema_prefix')}}_tuva_claims{% else %}_tuva_claims{%- endif -%}
 
   - name: procedure_pivot
-    description: Pivot of procedures from long (1 row/procedure) to wide (multiple procedure per row)
-    config:
-      schema: |
-        {%- if var('medicare_cclf_connector_schema',None) != None -%}{{ var('medicare_cclf_connector_schema') }}{%- elif var('tuva_schema_prefix',None) != None -%}{{var('tuva_schema_prefix')}}_tuva_claims{% else %}tuva_claims{%- endif -%}
-
-
-### final
-  - name: eligibility
     description: >
-      Member enrollment and demographic detail, converts member months to enrollment spans.
+      Pivot of procedures from long (1 row/procedure) to wide (multiple 
+      procedure per row).
     config:
       schema: |
-        {%- if var('medicare_cclf_connector_schema',None) != None -%}{{ var('medicare_cclf_connector_schema') }}{%- elif var('tuva_schema_prefix',None) != None -%}{{var('tuva_schema_prefix')}}_tuva_claims{% else %}tuva_claims{%- endif -%}
-
+        {%- if var('tuva_schema_prefix',None) != None -%}_{{var('tuva_schema_prefix')}}_tuva_claims{% else %}_tuva_claims{%- endif -%}
 
-  - name: medical_claim
-    description: Claim line level medical data
+### staging
+  - name: base_parta_diagnosis_code
+    description: Base model used as ref for pivot transformation.
     config:
       schema: |
-        {%- if var('medicare_cclf_connector_schema',None) != None -%}{{ var('medicare_cclf_connector_schema') }}{%- elif var('tuva_schema_prefix',None) != None -%}{{var('tuva_schema_prefix')}}_tuva_claims{% else %}tuva_claims{%- endif -%}
-    
+        {%- if var('tuva_schema_prefix',None) != None -%}_{{var('tuva_schema_prefix')}}_tuva_claims{% else %}_tuva_claims{%- endif -%}
 
-  - name: pharmacy_claim
-    description: Claim line level medical data
+  - name: base_parta_procedure_code
+    description: Base model used as ref for pivot transformation.
     config:
       schema: |
-        {%- if var('medicare_cclf_connector_schema',None) != None -%}{{ var('medicare_cclf_connector_schema') }}{%- elif var('tuva_schema_prefix',None) != None -%}{{var('tuva_schema_prefix')}}_tuva_claims{% else %}tuva_claims{%- endif -%}
+        {%- if var('tuva_schema_prefix',None) != None -%}_{{var('tuva_schema_prefix')}}_tuva_claims{% else %}_tuva_claims{%- endif -%}
diff --git a/models/final/eligibility.sql b/models/final/eligibility.sql
@@ -8,18 +8,19 @@ with demographics as (
 
     select
           bene_mbi_id
-        , {{ cast_string_or_varchar('bene_sex_cd') }} as bene_sex_cd
-        , {{ cast_string_or_varchar('bene_race_cd') }} as bene_race_cd
+        , cast(bene_sex_cd as {{ dbt.type_string() }} ) as bene_sex_cd
+        , cast(bene_race_cd as {{ dbt.type_string() }} ) as bene_race_cd
         , bene_dob
         , bene_death_dt
         , {{ try_to_cast_date('bene_member_month', 'YYYY-MM-DD') }} as bene_member_month
         , bene_dual_stus_cd
         , bene_mdcr_stus_cd
+        , bene_orgnl_entlmt_rsn_cd
         , bene_1st_name
         , bene_last_name
         , bene_line_1_adr
         , geo_zip_plc_name
-        , {{ cast_string_or_varchar('bene_fips_state_cd') }} as bene_fips_state_cd
+        , cast(bene_fips_state_cd as {{ dbt.type_string() }} ) as bene_fips_state_cd
         , bene_zip_cd
     from {{ source('medicare_cclf','beneficiary_demographics') }}
 
@@ -118,8 +119,8 @@ enrollment_span as (
 joined as (
 
     select
-          {{ cast_string_or_varchar('enrollment_span.bene_mbi_id') }} as patient_id
-        , {{ cast_string_or_varchar('enrollment_span.bene_mbi_id') }} as member_id
+          cast(enrollment_span.bene_mbi_id as {{ dbt.type_string() }} ) as patient_id
+        , cast(enrollment_span.bene_mbi_id as {{ dbt.type_string() }} ) as member_id
         , case demographics.bene_sex_cd
             when '0' then 'unknown'
             when '1' then 'male'
@@ -144,15 +145,16 @@ joined as (
         , enrollment_span.enrollment_end_date_last as enrollment_end_date
         , 'medicare' as payer
         , 'medicare' as payer_type
-        , {{ cast_string_or_varchar('demographics.bene_dual_stus_cd') }} as dual_status_code
-        , {{ cast_string_or_varchar('demographics.bene_mdcr_stus_cd') }} as medicare_status_code
-        , {{ cast_string_or_varchar('demographics.bene_1st_name') }} as first_name
-        , {{ cast_string_or_varchar('demographics.bene_last_name') }} as last_name
-        , {{ cast_string_or_varchar('demographics.bene_line_1_adr') }} as address
-        , {{ cast_string_or_varchar('demographics.geo_zip_plc_name') }} as city
-        , {{ cast_string_or_varchar('fips_state.ansi_fips_state_name') }} as state
-        , {{ cast_string_or_varchar('demographics.bene_zip_cd') }} as zip_code
-        , {{ cast_string_or_varchar('NULL') }} as phone
+        , cast(bene_orgnl_entlmt_rsn_cd as {{ dbt.type_string() }} ) as original_reason_entitlement_code
+        , cast(demographics.bene_dual_stus_cd as {{ dbt.type_string() }} ) as dual_status_code
+        , cast(demographics.bene_mdcr_stus_cd as {{ dbt.type_string() }} ) as medicare_status_code
+        , cast(demographics.bene_1st_name as {{ dbt.type_string() }} ) as first_name
+        , cast(demographics.bene_last_name as {{ dbt.type_string() }} ) as last_name
+        , cast(demographics.bene_line_1_adr as {{ dbt.type_string() }} ) as address
+        , cast(demographics.geo_zip_plc_name as {{ dbt.type_string() }} ) as city
+        , cast(fips_state.ansi_fips_state_name as {{ dbt.type_string() }} ) as state
+        , cast(demographics.bene_zip_cd as {{ dbt.type_string() }} ) as zip_code
+        , cast(NULL as {{ dbt.type_string() }} ) as phone
         , 'medicare cclf' as data_source
     from enrollment_span
          left join demographics