Merge pull request #10 from Unicon/create-tpdm-models-iteration-2

Create base/stage models (iteration)
Unicon · Aug 1, 2024 · c7b751b · c7b751b
2 parents 451d3ef + e34f43b
commit c7b751b
Show file tree

Hide file tree

Showing 34 changed files with 865 additions and 35 deletions.
diff --git a/macros/gen_skey.sql b/macros/gen_skey.sql
@@ -201,6 +201,25 @@
             'col_list': ['candidateIdentifier'],
             'annualize': True
         },
+
+        'k_applicant_profile': {
+            'reference_name': 'applicant_profile_reference',
+            'col_list': ['applicantProfileIdentifier'],
+            'annualize': True
+        },
+
+        'k_certification_exam': {
+            'reference_name': 'certification_exam_reference',
+            'col_list': ['certificationExamIdentifier', 'namespace'],
+            'annualize': True
+        },
+
+        'k_survey_section': {
+            'reference_name': 'certification_exam_reference',
+            'col_list': ['surveyIdentifier', 'namespace', 'survey_section_title'],
+            'annualize': True
+        },
+
         'k_person': {
             'reference_name': 'person_reference',
             'col_list': ['personId', 'sourceSystemDescriptor'],
@@ -297,6 +316,7 @@
                          'termDescriptor'],
             'annualize': True
         },
+
         'k_template': {
             'reference_name': '',
             'col_list': [],

diff --git a/models/staging/edfi_3/base/_edfi_3__base.yml b/models/staging/edfi_3/base/_edfi_3__base.yml
@@ -197,3 +197,9 @@ models:
   - name: base_ef3__survey_section_responses
     config:
       tags: ['survey']
+  - name: base_ef3__people
+    config:
+      tags: ['core']
+  - name: base_ef3__school_year_types
+    config:
+      tags: ['core']
diff --git a/models/staging/edfi_3/base/base_ef3__credentials.sql b/models/staging/edfi_3/base/base_ef3__credentials.sql
@@ -11,16 +11,18 @@ renamed as (
         filename,
         is_deleted,
 
-        v:id::string                   as record_guid,
-        v:credentialIdentifier::string as credential_id,
-        v:effectiveDate::date          as effective_date,
-        v:expirationDate::date         as expiration_date,
-        v:issuanceDate::date           as issuance_date,
-        v:namespace::string            as namespace,
+        v:id::string as record_guid,
+        -- identity components
+        v:credentialIdentifier::string                                                as credential_id,
+        {{ extract_descriptor('v:stateOfIssueStateAbbreviationDescriptor::string') }} as state_of_issue_state_abbreviation,
+        -- non-identity components
+        v:effectiveDate::date  as effective_date,
+        v:expirationDate::date as expiration_date,
+        v:issuanceDate::date   as issuance_date,
+        v:namespace::string    as namespace,
         -- descriptors
         {{ extract_descriptor('v:credentialFieldDescriptor::string') }}               as credential_field,
         {{ extract_descriptor('v:credentialTypeDescriptor::string') }}                as credential_type,
-        {{ extract_descriptor('v:stateOfIssueStateAbbreviationDescriptor::string') }} as state_of_issue_state_abbreviation,
         {{ extract_descriptor('v:teachingCredentialDescriptor::string') }}            as teaching_credential,
         {{ extract_descriptor('v:teachingCredentialBasisDescriptor::string') }}       as teaching_credential_basis,
         -- unnested lists

diff --git a/models/staging/edfi_3/base/base_ef3__people.sql b/models/staging/edfi_3/base/base_ef3__people.sql
@@ -0,0 +1,20 @@
+with people as (
+    {{ source_edfi3('people') }}
+),
+renamed as (
+    select 
+        tenant_code,
+        api_year,
+        pull_timestamp,
+        last_modified_timestamp,
+        file_row_number,
+        filename,
+        is_deleted,
+
+        v:id::string as record_guid,
+        -- identity components
+        v:personId::int as person_id,
+        {{ extract_descriptor('v:sourceSystemDescriptor::string') }} as source_system
+    from people
+)
+select * from renamed
diff --git a/models/staging/edfi_3/base/base_ef3__post_secondary_institutions.sql b/models/staging/edfi_3/base/base_ef3__post_secondary_institutions.sql
@@ -11,8 +11,10 @@ renamed as (
         filename,
         is_deleted,
 
-        v:id::string                      as record_guid,
+        v:id::string as record_guid,
+        -- identity components
         v:postSecondaryInstitutionId::int as post_secondary_institution_id,
+        -- non-identity components
         v:nameOfInstitution::string       as name_of_institution,
         v:shortNameOfInstitution::string  as short_name_of_institution,
         v:webSite::string                 as web_site,

diff --git a/models/staging/edfi_3/base/base_ef3__school_year_types.sql b/models/staging/edfi_3/base/base_ef3__school_year_types.sql
@@ -0,0 +1,22 @@
+with school_year_types as (
+    {{ source_edfi3('school_year_types') }}
+),
+renamed as (
+    select
+        tenant_code,
+        api_year,
+        pull_timestamp,
+        last_modified_timestamp,
+        filename,
+        file_row_number,
+        is_deleted,
+
+        v:id::string as record_guid,
+        -- identity components
+        v:schoolYear::string as school_year,
+        -- non-identity components
+        v:currentSchoolYear::string     as current_school_year,
+        v:schoolYearDescription::string as school_year_description
+    from school_year_types
+)
+select * from renamed
diff --git a/models/staging/edfi_3/base/base_ef3__survey_section_responses.sql b/models/staging/edfi_3/base/base_ef3__survey_section_responses.sql
@@ -11,10 +11,14 @@ renamed as (
         filename,
         is_deleted,
 
-        v:id::string                                               as record_guid,
-        v:surveyResponseReference:surveyResponseIdentifier::string as survey_response_id,
+        v:id::string as record_guid,
+        -- identity components
+        v:surveyResponseReference:namespace::string                as namespace,
         v:surveySectionReference:surveyIdentifier::string          as survey_id,
-        v:sectionRating::float                                     as section_rating,
+        v:surveyResponseReference:surveyResponseIdentifier::string as survey_response_id,
+        v:surveySectionReference:surveySectionTitle::string        as survey_section_title,
+        -- non-identity components
+        v:sectionRating::float as section_rating,
         -- references
         v:surveyResponseReference as survey_response_reference,
         v:surveySectionReference  as survey_section_reference

diff --git a/models/staging/edfi_3/base/base_ef3__survey_sections.sql b/models/staging/edfi_3/base/base_ef3__survey_sections.sql
@@ -11,8 +11,10 @@ renamed as (
         filename,
         is_deleted,
 
-        v:id::string                                as record_guid,
-        v:surveyReference:surveyIdentifier::string  as survey_id,
+        v:id::string as record_guid,
+        -- identity components
+        v:surveyReference::surveyIdentifier::string as survey_id,
+        v:surveyReference::namespace::string        as namespace,
         v:surveySectionTitle::string                as survey_section_title,
         -- references
         v:surveyReference as survey_reference

diff --git a/models/staging/edfi_3/stage/_edfi_3__stage.yml b/models/staging/edfi_3/stage/_edfi_3__stage.yml
@@ -127,6 +127,11 @@ referential_integrity_tests:
         field: k_survey_response
         tags: ['ref_integrity']
 
+  - k_survey_section: &ref_k_survey_section
+    - relationships:
+        to: ref('stg_ef3__survey_sections')
+        field: k_survey_section
+
   - k_person: &ref_k_person
     - relationships:
         to: ref('stg_ef3__people')
@@ -853,3 +858,43 @@ models:
     config: 
       tags: ['survey']
       enabled: "{{ var('src:domain:survey:enabled', True) }}"
+
+  - name: stg_ef3__people
+    config:
+      tags: ['core']
+
+  - name: stg_ef3__credentials
+    config:
+      tags: ['core']
+    columns:
+      - name: k_student_academic_record
+        tests:
+          *ref_k_student_academic_record
+
+  - name: stg_ef3__post_secondary_institutions
+    config:
+      tags: ['core']
+
+  - name: stg_ef3__survey_sections
+    config:
+      tags: ['core']
+    columns:
+      - name: k_survey
+        tests:
+          *ref_k_survey
+
+  - name: stg_ef3__survey_section_responses
+    config:
+      tags: ['core']
+    columns:
+      - name: k_survey_response
+        tests:
+          *ref_k_survey_response
+      - name: k_survey_section
+        tests:
+          *ref_k_survey_section
+
+  - name: stg_ef3__school_year_types
+    config:
+      tags: ['core']
+
diff --git a/models/staging/edfi_3/stage/stg_ef3__credentials.sql b/models/staging/edfi_3/stage/stg_ef3__credentials.sql
@@ -0,0 +1,29 @@
+with credentials as (
+    select * from {{ ref('base_tpdm__credentials') }}
+    where not is_deleted
+),
+keyed as (
+    select 
+        {{ dbt_utils.surrogate_key(
+            ['tenant_code',
+            'api_year',
+            'lower(namespace)',
+            'lower(person_id)',
+            'lower(source_system)',
+            'lower(survey_id)',
+            'lower(survey_response_id)']
+        ) }} as k_credential,
+        {{ gen_skey('k_student_academic_record')}},
+        credentials.*
+        {{ extract_extension(model_name=this.name, flatten=True) }}
+    from credentials
+),
+deduped as (
+    {{
+        dbt_utils.deduplicate(
+            relation='keyed',
+            partition_by='k_credential',
+            order_by='pull_timestamp desc')
+    }}
+)
+select * from deduped
diff --git a/models/staging/edfi_3/stage/stg_ef3__people.sql b/models/staging/edfi_3/stage/stg_ef3__people.sql
@@ -0,0 +1,25 @@
+with base_people as (
+    select * from {{ ref('base_ef3__people') }}
+    where not is_deleted
+),
+keyed as (
+    select 
+        {{ dbt_utils.surrogate_key(
+            ['tenant_code',
+            'api_year',
+            'person_id',
+            'lower(source_system)']
+        ) }} as k_person,
+        base_people.*
+        {{ extract_extension(model_name=this.name, flatten=True) }}
+    from base_people
+),
+deduped as (
+    {{
+        dbt_utils.deduplicate(
+            relation='keyed',
+            partition_by='k_person',
+            order_by='pull_timestamp desc')
+    }}
+)
+select * from deduped
diff --git a/models/staging/edfi_3/stage/stg_ef3__post_secondary_institutions.sql b/models/staging/edfi_3/stage/stg_ef3__post_secondary_institutions.sql
@@ -0,0 +1,24 @@
+with post_secondary_institutions as (
+    select * from {{ ref('base_tpdm__post_secondary_institutions') }}
+    where not is_deleted
+),
+keyed as (
+    select 
+        {{ dbt_utils.surrogate_key(
+            ['tenant_code',
+            'api_year',
+            'post_secondary_institution_id']
+        ) }} as k_post_secondary_institution,
+        post_secondary_institutions.*
+        {{ extract_extension(model_name=this.name, flatten=True) }}
+    from post_secondary_institutions
+),
+deduped as (
+    {{
+        dbt_utils.deduplicate(
+            relation='keyed',
+            partition_by='k_post_secondary_institution',
+            order_by='pull_timestamp desc')
+    }}
+)
+select * from deduped
diff --git a/models/staging/edfi_3/stage/stg_ef3__school_year_types.sql b/models/staging/edfi_3/stage/stg_ef3__school_year_types.sql
@@ -0,0 +1,24 @@
+with school_year_types as (
+    select * from {{ ref('base_ef3__school_year_types') }}
+    where not is_deleted
+),
+keyed as (
+    select 
+        {{ dbt_utils.surrogate_key(
+            ['tenant_code',
+            'api_year',
+            'lower(school_year)']
+        ) }} as k_school_year_type,
+        school_year_types.*
+        {{ extract_extension(model_name=this.name, flatten=True) }}
+    from school_year_types
+),
+deduped as (
+    {{
+        dbt_utils.deduplicate(
+            relation='keyed',
+            partition_by='k_school_year_type',
+            order_by='pull_timestamp desc')
+    }}
+)
+select * from deduped
diff --git a/models/staging/edfi_3/stage/stg_ef3__survey_section_responses.sql b/models/staging/edfi_3/stage/stg_ef3__survey_section_responses.sql
@@ -0,0 +1,28 @@
+with survey_section_responses as (
+    select * from {{ ref('base_ef3__survey_section_responses') }}
+    where not is_deleted
+),
+keyed as (
+    select 
+        {{ dbt_utils.surrogate_key(
+            ['tenant_code',
+            'api_year',
+            'lower(survey_id)',
+            'lower(namespace)',
+            'lower(survey_section_title)']
+        ) }} as k_survey_section_response,
+        {{ gen_skey('k_survey_response') }},
+        {{ gen_skey('k_survey_section') }},
+        survey_section_responses.*
+        {{ extract_extension(model_name=this.name, flatten=True) }}
+    from survey_section_responses
+),
+deduped as (
+    {{
+        dbt_utils.deduplicate(
+            relation='keyed',
+            partition_by='k_survey_section_response',
+            order_by='pull_timestamp desc')
+    }}
+)
+select * from deduped
diff --git a/models/staging/edfi_3/stage/stg_ef3__survey_sections.sql b/models/staging/edfi_3/stage/stg_ef3__survey_sections.sql
@@ -0,0 +1,27 @@
+with survey_sections as (
+    select * from {{ ref('base_ef3__survey_sections') }}
+    where not is_deleted
+),
+keyed as (
+    select 
+        {{ dbt_utils.surrogate_key(
+            ['tenant_code',
+            'api_year',
+            'lower(survey_id)',
+            'lower(namespace)',
+            'lower(survey_section_title)']
+        ) }} as k_survey_section,
+        {{ gen_skey('k_survey') }},
+        survey_sections.*
+        {{ extract_extension(model_name=this.name, flatten=True) }}
+    from survey_sections
+),
+deduped as (
+    {{
+        dbt_utils.deduplicate(
+            relation='keyed',
+            partition_by='k_survey_section',
+            order_by='pull_timestamp desc')
+    }}
+)
+select * from deduped