Merge pull request elementary-data#941 from elementary-data/ele-966-a…

…dd-exposure-data

Ele 966 add exposure data

.gitignore

@@ -97,6 +97,4 @@ venv/
 .vscode/
 
 # elementary outputs
-elementary_output.json
-elementary_report.html
-edr.log
+edr_target/

elementary/monitor/api/models/models.py

@@ -208,6 +208,7 @@ def _normalize_dbt_artifact_dict(
         normalized_artifact["normalized_full_path"] = self._normalize_artifact_path(
             artifact
         )
+        normalized_artifact["fqn"] = self._fqn(artifact)
 
         return schema_to_normalized_schema_map[type(artifact)](**normalized_artifact)
 
@@ -235,3 +236,22 @@ def _normalize_artifact_path(
             splited_artifact_path.insert(0, artifact.package_name)
 
         return os.path.sep.join(splited_artifact_path)
+
+    @classmethod
+    def _fqn(
+        cls,
+        artifact: Union[ArtifactSchemaType],
+    ) -> str:
+        if isinstance(artifact, ExposureSchema):
+            path = (artifact.meta or {}).get("path")
+            name = artifact.label or artifact.name
+            fqn = f"{path}/{name}" if path else name
+            return fqn
+
+        fqn = (
+            f"{artifact.database_name}.{artifact.schema_name}.{artifact.table_name}"
+            if artifact.database_name is not None and artifact.schema_name is not None
+            else artifact.table_name
+        )
+
+        return fqn.lower()

elementary/monitor/api/models/schema.py

@@ -20,6 +20,7 @@ class NormalizedArtifactSchema(ExtendedBaseModel):
     # Currently its model_name to match the CLI UI.
     model_name: str
     normalized_full_path: str
+    fqn: str
 
     @validator("tags", pre=True)
     def load_tags(cls, tags):

elementary/monitor/dbt_project/macros/get_exposures.sql

@@ -1,11 +1,15 @@
 {% macro get_exposures() %}
     {% set dbt_exposures_relation = ref('elementary', 'dbt_exposures') %}
+    {% set label_column_exists = elementary.column_exists_in_relation(dbt_exposures_relation, 'label') %}
     {%- if elementary.relation_exists(dbt_exposures_relation) -%}
         --{# TODO: should we group by #}
         {% set get_exposures_query %}
               with dbt_artifacts_exposures as (
                 select
                   name,
+                  {% if label_column_exists %}
+                    label,
+                  {% endif %}
                   unique_id,
                   url,
                   type,
@@ -15,6 +19,7 @@
                   tags,
                   package_name,
                   description,
+                  meta,
                   original_path as full_path
                 from {{ dbt_exposures_relation }}
               )

elementary/monitor/dbt_project/packages.yml

@@ -1,5 +1,5 @@
 packages:
   - package: dbt-labs/dbt_utils
     version: [">=0.8.0", "<0.9.0"]
-  - package: elementary-data/elementary
-    version: 0.8.1
+  - git: "https://github.com/elementary-data/dbt-data-reliability.git"
+    revision: 8cb5570b562b044c58c131e4b9178e14fc3bd29a

elementary/monitor/fetchers/models/schema.py

@@ -46,6 +46,10 @@ def load_owners(cls, owners):
     def format_full_path_sep(cls, full_path: str) -> str:
         return posixpath.sep.join(full_path.split(os.path.sep))
 
+    @validator("meta", pre=True)
+    def load_meta(cls, meta):
+        return cls._load_var_to_dict(meta)
+
 
 ArtifactSchemaType = TypeVar("ArtifactSchemaType", bound=ArtifactSchema)