Merge pull request #375 from elementary-data/ele-1061-databricks-enab…

…ledfalse-gets-override-by-user-config Made databricks avoid running info schema models even if user `enabled: true`
elementary-data · Jun 20, 2023 · 057b343 · 057b343
2 parents 62d41e7 + 0393463
commit 057b343
Show file tree

Hide file tree

Showing 9 changed files with 11 additions and 36 deletions.
diff --git a/macros/edr/metadata_collection/get_columns_by_schemas.sql b/macros/edr/metadata_collection/get_columns_by_schemas.sql
@@ -2,6 +2,6 @@
     {%- if configured_schemas | length > 0 -%}
         {{ elementary.union_macro_queries(configured_schemas, elementary.get_columns_from_information_schema) }}
     {%- else %}
-        {{ elementary.empty_table([('full_table_name', 'string'), ('database_name', 'string'), ('schema_name', 'string'), ('table_name', 'string'), ('column_name', 'string'), ('data_type', 'string')]) }}
+        {{ elementary.get_empty_columns_from_information_schema_table() }}
     {%- endif %}
 {% endmacro %}
diff --git a/macros/edr/metadata_collection/get_columns_from_information_schema.sql b/macros/edr/metadata_collection/get_columns_from_information_schema.sql
@@ -43,4 +43,12 @@
     from information_schema.columns
     where upper(table_schema) = upper('{{ schema_name }}')
 
-{% endmacro %}
+{% endmacro %}
+
+{% macro spark__get_columns_from_information_schema(database_name, schema_name) %}
+    {{ elementary.get_empty_columns_from_information_schema_table() }}
+{% endmacro %}
+
+{% macro get_empty_columns_from_information_schema_table() %}
+    {{ elementary.empty_table([('full_table_name', 'string'), ('database_name', 'string'), ('schema_name', 'string'), ('table_name', 'string'), ('column_name', 'string'), ('data_type', 'string')]) }}
+{% endmacro %}
diff --git a/macros/edr/system/system_utils/get_config_var.sql b/macros/edr/system/system_utils/get_config_var.sql
@@ -32,7 +32,6 @@
     'disable_tests_results': false,
     'disable_dbt_artifacts_autoupload': false,
     'disable_dbt_invocation_autoupload': false,
-    'enable_dbt_columns': true,
     'disable_skipped_model_alerts': true,
     'disable_skipped_test_alerts': true,
     'dbt_artifacts_chunk_size': 5000,

diff --git a/models/edr/alerts/alerts_schema_changes.sql b/models/edr/alerts/alerts_schema_changes.sql
@@ -2,7 +2,6 @@
   config(
     materialized = 'view',
     bind=False,
-    enabled = target.type != 'databricks' and target.type != 'spark' | as_bool()
   )
 }}
 

diff --git a/models/edr/data_monitoring/schema_changes/schema_columns_snapshot.sql b/models/edr/data_monitoring/schema_changes/schema_columns_snapshot.sql
@@ -2,7 +2,6 @@
   config(
     materialized='incremental',
     unique_key = 'column_state_id',
-    enabled = target.type != 'databricks' and target.type != 'spark' | as_bool(),
     on_schema_change = 'append_new_columns',
     full_refresh=elementary.get_config_var('elementary_full_refresh'),
     meta={

diff --git a/models/edr/dbt_artifacts/dbt_columns.sql b/models/edr/dbt_artifacts/dbt_columns.sql
@@ -1,7 +1,6 @@
 {{
   config(
     materialized = 'view' if not var('sync', false) else 'table',
-    enabled = elementary.get_config_var('enable_dbt_columns') and target.type != 'databricks' and target.type != 'spark' | as_bool()
   )
 }}
 

diff --git a/models/edr/metadata_store/filtered_information_schema_columns.sql b/models/edr/metadata_store/filtered_information_schema_columns.sql
@@ -1,7 +1,6 @@
 {{
   config(
     materialized = 'view',
-    enabled = target.type != 'databricks' and target.type != 'spark' | as_bool()
   )
 }}
 
@@ -13,4 +12,4 @@ with filtered_information_schema_columns as (
 
 select *
 from filtered_information_schema_columns
-where full_table_name is not null
+where full_table_name is not null
diff --git a/models/edr/metadata_store/filtered_information_schema_tables.sql b/models/edr/metadata_store/filtered_information_schema_tables.sql
diff --git a/models/elementary_tests.yml b/models/elementary_tests.yml
@@ -35,9 +35,3 @@ models:
       Queries the columns view from the information schema of the schemas in the project.
       This view is generated using an adapter specific macro, as information schema is different between platforms.
       This is a view to make the work with the information schema more convenient.
-
-  - name: filtered_information_schema_tables
-    description: >
-      Queries the tables and schemas views from the information schema of the schemas in the project.
-      This view is generated using an adapter specific macro, as information schema is different between platforms.
-      This is a view to make the work with the information schema more convenient.
-Original file line number
+Diff line change
@@ Expand Up / @@ -2,7 +2,6 @@ @@
       config(
         materialized = 'view',
         bind=False,
-        enabled = target.type != 'databricks' and target.type != 'spark' | as_bool()
       )
     }}
@@ Expand Down @@