Merge pull request #14 from mattiasthalen/add-rarity-to-dv

Add rarity to data vault

README.md

@@ -49,6 +49,7 @@ erDiagram
     raw__hub__card_set ||--|{ raw__sat__card_set: ""
     raw__hub__card_type ||--|{ raw__sat__card_type: ""
     raw__hub__class ||--|{ raw__sat__class: ""
+    raw__hub__rarity ||--|{ raw__sat__rarity: ""
     raw__hub__spell_school ||--|{ raw__sat__spell_school: ""
     
     raw__link__card__card_set }|--|| raw__hub__card: ""
@@ -63,6 +64,10 @@ erDiagram
     raw__link__card__class }|--|| raw__hub__class: ""
     raw__link__card__class ||--|{ raw__sat_eff__card__class: ""
     
+    raw__link__card__rarity }|--|| raw__hub__card: ""
+    raw__link__card__rarity }|--|| raw__hub__rarity: ""
+    raw__link__card__rarity ||--|{ raw__sat_eff__card__rarity: ""
+    
     raw__link__card__spell_school }|--|| raw__hub__card: ""
     raw__link__card__spell_school }|--|| raw__hub__spell_school: ""
     raw__link__card__spell_school ||--|{ raw__sat_eff__card__spell_school: ""

models/silver/raw_vault/raw__hub__rarity.sql

@@ -0,0 +1,55 @@
+/* Data vault hub model for the Hearthstone rarities */
+MODEL (
+  name silver.raw_vault.raw__hub__rarity,
+  kind INCREMENTAL_BY_UNIQUE_KEY (
+    unique_key hash_key__rarity_bk
+  ),
+  columns (
+    rarity_bk TEXT,
+    hash_key__rarity_bk TEXT,
+    _sqlmesh__record_source TEXT,
+    _sqlmesh__extracted_at TIMESTAMP,
+    _sqlmesh__loaded_at TIMESTAMP
+  )
+);
+
+WITH business_keys AS (
+  SELECT
+    0 AS source,
+    rarity_bk,
+    hash_key__rarity_bk,
+    _sqlmesh__record_source,
+    _sqlmesh__extracted_at,
+    _sqlmesh__loaded_at
+  FROM silver.staging.dv_stg__hearthstone__rarities
+  UNION ALL
+  SELECT
+    1 AS source,
+    rarity_id AS rarity_bk,
+    hash_key__rarity_id AS hash_key__rarity_bk,
+    _sqlmesh__record_source,
+    _sqlmesh__extracted_at,
+    _sqlmesh__loaded_at
+  FROM silver.staging.dv_stg__hearthstone__rarities
+  UNION ALL
+  SELECT
+    2 AS source,
+    rarity_bk,
+    hash_key__rarity_bk,
+    _sqlmesh__record_source,
+    _sqlmesh__extracted_at,
+    _sqlmesh__loaded_at
+  FROM silver.staging.dv_stg__hearthstone__cards
+), deduplicated AS (
+  SELECT
+    *
+    EXCLUDE (source)
+  FROM business_keys
+  QUALIFY
+    ROW_NUMBER() OVER (PARTITION BY rarity_bk ORDER BY source, _sqlmesh__loaded_at) = 1
+)
+SELECT
+  *
+FROM deduplicated
+ANTI JOIN silver.raw_vault.raw__hub__rarity
+  USING (hash_key__rarity_bk)

models/silver/raw_vault/raw__link__card__rarity.sql

@@ -0,0 +1,22 @@
+/* Data vault link model for the Hearthstone cards & rarities */
+MODEL (
+  name silver.raw_vault.raw__link__card__rarity,
+  kind SCD_TYPE_2_BY_COLUMN (
+    unique_key hash_key__card_bk__rarity_bk,
+    columns [hash_key__card_bk__rarity_bk],
+    execution_time_as_valid_from TRUE,
+    disable_restatement FALSE,
+    valid_to_name _sqlmesh__valid_to,
+    valid_from_name _sqlmesh__valid_from
+  ),
+  allow_partials TRUE
+);
+
+SELECT
+  hash_key__card_bk__rarity_bk,
+  hash_key__card_bk,
+  hash_key__rarity_bk,
+  _sqlmesh__record_source,
+  _sqlmesh__extracted_at,
+  _sqlmesh__loaded_at
+FROM silver.staging.dv_stg__hearthstone__cards

models/silver/raw_vault/raw__sat__rarity.sql

@@ -0,0 +1,24 @@
+/* Data vault satellite model for the Hearthstone spell schools */
+MODEL (
+  name silver.raw_vault.raw__sat__rarity,
+  kind SCD_TYPE_2_BY_COLUMN (
+    unique_key hash_key__rarity_bk,
+    columns [_sqlmesh__hash_diff],
+    valid_to_name _sqlmesh__valid_to,
+    valid_from_name _sqlmesh__valid_from
+  ),
+  allow_partials TRUE
+);
+
+SELECT
+  hash_key__rarity_bk,
+  slug,
+  id,
+  name,
+  craftingCost,
+  dustValue,
+  _sqlmesh__hash_diff,
+  _sqlmesh__record_source,
+  _sqlmesh__extracted_at,
+  _sqlmesh__loaded_at
+FROM silver.staging.dv_stg__hearthstone__rarities

models/silver/raw_vault/raw__sat_eff__card__rarity.sql

@@ -0,0 +1,18 @@
+/* Data vault effectivity satellite model for the Hearthstone cards & rarities */
+MODEL (
+  name silver.raw_vault.raw__sat_eff__card__rarity,
+  kind SCD_TYPE_2_BY_COLUMN (
+    unique_key hash_key__card_bk__rarity_bk,
+    columns [hash_key__card_bk__rarity_bk],
+    valid_to_name _sqlmesh__valid_to,
+    valid_from_name _sqlmesh__valid_from
+  ),
+  allow_partials TRUE
+);
+
+SELECT
+  hash_key__card_bk__rarity_bk,
+  _sqlmesh__record_source,
+  _sqlmesh__extracted_at,
+  _sqlmesh__loaded_at
+FROM silver.staging.dv_stg__hearthstone__cards

models/silver/staging/dv_stg__hearthstone__rarities.sql

@@ -0,0 +1,46 @@
+/* Data vault staging model for the Hearthstone rarities */
+MODEL (
+  name silver.staging.dv_stg__hearthstone__rarities,
+  kind FULL
+);
+
+WITH source AS (
+  SELECT
+    *,
+    @execution_ts::TIMESTAMP AS _sqlmesh__loaded_at
+  FROM bronze.raw.raw__hearthstone__rarities
+), keys AS (
+  SELECT
+    *,
+    id::TEXT AS rarity_id,
+    slug::TEXT AS rarity_bk
+  FROM source
+), ghost_record AS (
+  SELECT
+    keys.*
+    REPLACE (ghost._sqlmesh__record_source AS _sqlmesh__record_source, ghost._sqlmesh__loaded_at AS _sqlmesh__loaded_at)
+  FROM (
+    SELECT
+      'GHOST_RECORD' AS _sqlmesh__record_source,
+      '0001-01-01 00:00:00'::TIMESTAMP AS _sqlmesh__extracted_at,
+      '0001-01-01 00:00:00'::TIMESTAMP AS _sqlmesh__loaded_at
+  ) AS ghost
+  LEFT JOIN keys
+    ON ghost._sqlmesh__record_source = keys._sqlmesh__record_source
+    AND ghost._sqlmesh__extracted_at = keys._sqlmesh__extracted_at
+    AND ghost._sqlmesh__loaded_at = keys._sqlmesh__loaded_at
+  UNION ALL
+  SELECT
+    *
+  FROM keys
+), hashes AS (
+  SELECT
+    *,
+    @generate_surrogate_key(rarity_id, hash_function := 'SHA256') AS hash_key__rarity_id,
+    @generate_surrogate_key(rarity_bk, hash_function := 'SHA256') AS hash_key__rarity_bk,
+    @generate_surrogate_key(rarity_bk, slug, id, name, craftingCost, dustValue, hash_function := 'SHA256') AS _sqlmesh__hash_diff
+  FROM ghost_record
+)
+SELECT
+  *
+FROM hashes