diff --git a/README.md b/README.md index 8853ba0..f78bdc8 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,7 @@ erDiagram erDiagram raw__hub__card ||--|{ raw__sat__card: "" raw__hub__card_set ||--|{ raw__sat__card_set: "" + raw__hub__card_type ||--|{ raw__sat__card_type: "" raw__hub__class ||--|{ raw__sat__class: "" raw__hub__spell_school ||--|{ raw__sat__spell_school: "" @@ -54,6 +55,10 @@ erDiagram raw__link__card__card_set }|--|| raw__hub__card_set: "" raw__link__card__card_set ||--|{ raw__sat_eff__card__card_set: "" + raw__link__card__card_type }|--|| raw__hub__card: "" + raw__link__card__card_type }|--|| raw__hub__card_type: "" + raw__link__card__card_type ||--|{ raw__sat_eff__card__card_type: "" + raw__link__card__class }|--|| raw__hub__card: "" raw__link__card__class }|--|| raw__hub__class: "" raw__link__card__class ||--|{ raw__sat_eff__card__class: "" diff --git a/models/silver/raw_vault/raw__hub__card_type.sql b/models/silver/raw_vault/raw__hub__card_type.sql new file mode 100644 index 0000000..a1b08c4 --- /dev/null +++ b/models/silver/raw_vault/raw__hub__card_type.sql @@ -0,0 +1,55 @@ +/* Data vault hub model for the Hearthstone card types */ +MODEL ( + name silver.raw_vault.raw__hub__card_type, + kind INCREMENTAL_BY_UNIQUE_KEY ( + unique_key hash_key__card_type_bk + ), + columns ( + card_type_bk TEXT, + hash_key__card_type_bk TEXT, + _sqlmesh__record_source TEXT, + _sqlmesh__extracted_at TIMESTAMP, + _sqlmesh__loaded_at TIMESTAMP + ) +); + +WITH business_keys AS ( + SELECT + 0 AS source, + card_type_bk, + hash_key__card_type_bk, + _sqlmesh__record_source, + _sqlmesh__extracted_at, + _sqlmesh__loaded_at + FROM silver.staging.dv_stg__hearthstone__types + UNION ALL + SELECT + 1 AS source, + card_type_id AS card_type_bk, + hash_key__card_type_id AS hash_key__card_type_bk, + _sqlmesh__record_source, + _sqlmesh__extracted_at, + _sqlmesh__loaded_at + FROM silver.staging.dv_stg__hearthstone__types + UNION ALL + SELECT + 2 AS source, + card_type_bk, + hash_key__card_type_bk, + _sqlmesh__record_source, + _sqlmesh__extracted_at, + _sqlmesh__loaded_at + FROM silver.staging.dv_stg__hearthstone__cards +), deduplicated AS ( + SELECT + * + EXCLUDE (source) + FROM business_keys + QUALIFY + ROW_NUMBER() OVER (PARTITION BY card_type_bk ORDER BY source, _sqlmesh__loaded_at) = 1 +) +SELECT + * +FROM deduplicated +ANTI JOIN silver.raw_vault.raw__hub__card_type + USING (hash_key__card_type_bk) \ No newline at end of file diff --git a/models/silver/raw_vault/raw__link__card__card_type.sql b/models/silver/raw_vault/raw__link__card__card_type.sql new file mode 100644 index 0000000..a95ec7a --- /dev/null +++ b/models/silver/raw_vault/raw__link__card__card_type.sql @@ -0,0 +1,22 @@ +/* Data vault link model for the Hearthstone cards & card types */ +MODEL ( + name silver.raw_vault.raw__link__card__card_type, + kind SCD_TYPE_2_BY_COLUMN ( + unique_key hash_key__card_bk__card_type_bk, + columns [hash_key__card_bk__card_type_bk], + execution_time_as_valid_from TRUE, + disable_restatement FALSE, + valid_to_name _sqlmesh__valid_to, + valid_from_name _sqlmesh__valid_from + ), + allow_partials TRUE +); + +SELECT + hash_key__card_bk__card_type_bk, + hash_key__card_bk, + hash_key__card_type_bk, + _sqlmesh__record_source, + _sqlmesh__extracted_at, + _sqlmesh__loaded_at +FROM silver.staging.dv_stg__hearthstone__cards \ No newline at end of file diff --git a/models/silver/raw_vault/raw__sat__card_type.sql b/models/silver/raw_vault/raw__sat__card_type.sql new file mode 100644 index 0000000..490bf4f --- /dev/null +++ b/models/silver/raw_vault/raw__sat__card_type.sql @@ -0,0 +1,23 @@ +/* Data vault satellite model for the Hearthstone card types */ +MODEL ( + name silver.raw_vault.raw__sat__card_type, + kind SCD_TYPE_2_BY_COLUMN ( + unique_key hash_key__card_type_bk, + columns [_sqlmesh__hash_diff], + valid_to_name _sqlmesh__valid_to, + valid_from_name _sqlmesh__valid_from + ), + allow_partials TRUE +); + +SELECT + hash_key__card_type_bk, + slug, + id, + name, + gameModes, + _sqlmesh__hash_diff, + _sqlmesh__record_source, + _sqlmesh__extracted_at, + _sqlmesh__loaded_at +FROM silver.staging.dv_stg__hearthstone__types \ No newline at end of file diff --git a/models/silver/raw_vault/raw__sat__class.sql b/models/silver/raw_vault/raw__sat__class.sql index 2fe5e01..0f5c268 100644 --- a/models/silver/raw_vault/raw__sat__class.sql +++ b/models/silver/raw_vault/raw__sat__class.sql @@ -19,4 +19,4 @@ SELECT _sqlmesh__record_source, _sqlmesh__extracted_at, _sqlmesh__loaded_at -FROM silver.staging.dv_stg__hearthstone__cards \ No newline at end of file +FROM silver.staging.dv_stg__hearthstone__classes \ No newline at end of file diff --git a/models/silver/raw_vault/raw__sat_eff__card__card_type.sql b/models/silver/raw_vault/raw__sat_eff__card__card_type.sql new file mode 100644 index 0000000..31c2e81 --- /dev/null +++ b/models/silver/raw_vault/raw__sat_eff__card__card_type.sql @@ -0,0 +1,18 @@ +/* Data vault effectivity satellite model for the Hearthstone cards & card types */ +MODEL ( + name silver.raw_vault.raw__sat_eff__card__card_type, + kind SCD_TYPE_2_BY_COLUMN ( + unique_key hash_key__card_bk__card_type_bk, + columns [hash_key__card_bk__card_type_bk], + valid_to_name _sqlmesh__valid_to, + valid_from_name _sqlmesh__valid_from + ), + allow_partials TRUE +); + +SELECT + hash_key__card_bk__card_type_bk, + _sqlmesh__record_source, + _sqlmesh__extracted_at, + _sqlmesh__loaded_at +FROM silver.staging.dv_stg__hearthstone__cards \ No newline at end of file diff --git a/models/silver/staging/dv_stg__hearthstone__types.sql b/models/silver/staging/dv_stg__hearthstone__types.sql new file mode 100644 index 0000000..acbff67 --- /dev/null +++ b/models/silver/staging/dv_stg__hearthstone__types.sql @@ -0,0 +1,46 @@ +/* Data vault staging model for the Hearthstone card types */ +MODEL ( + name silver.staging.dv_stg__hearthstone__types, + kind FULL +); + +WITH source AS ( + SELECT + *, + @execution_ts::TIMESTAMP AS _sqlmesh__loaded_at + FROM bronze.raw.raw__hearthstone__types +), keys AS ( + SELECT + *, + id::TEXT AS card_type_id, + slug::TEXT AS card_type_bk + FROM source +), ghost_record AS ( + SELECT + keys.* + REPLACE (ghost._sqlmesh__record_source AS _sqlmesh__record_source, ghost._sqlmesh__loaded_at AS _sqlmesh__loaded_at) + FROM ( + SELECT + 'GHOST_RECORD' AS _sqlmesh__record_source, + '0001-01-01 00:00:00'::TIMESTAMP AS _sqlmesh__extracted_at, + '0001-01-01 00:00:00'::TIMESTAMP AS _sqlmesh__loaded_at + ) AS ghost + LEFT JOIN keys + ON ghost._sqlmesh__record_source = keys._sqlmesh__record_source + AND ghost._sqlmesh__extracted_at = keys._sqlmesh__extracted_at + AND ghost._sqlmesh__loaded_at = keys._sqlmesh__loaded_at + UNION ALL + SELECT + * + FROM keys +), hashes AS ( + SELECT + *, + @generate_surrogate_key(card_type_id, hash_function := 'SHA256') AS hash_key__card_type_id, + @generate_surrogate_key(card_type_bk, hash_function := 'SHA256') AS hash_key__card_type_bk, + @generate_surrogate_key(card_type_bk, slug, id, name, gameModes, hash_function := 'SHA256') AS _sqlmesh__hash_diff + FROM ghost_record +) +SELECT + * +FROM hashes \ No newline at end of file