From 6900df8d6d06f8b25a30c65ddab64bde09aa7b3a Mon Sep 17 00:00:00 2001 From: Alex Roy Date: Wed, 13 Mar 2024 12:20:01 -0400 Subject: [PATCH 1/6] add flag to exclude unverified collection items --- dbtmetabase/__main__.py | 9 +++++++++ dbtmetabase/_exposures.py | 3 +++ dbtmetabase/metabase.py | 11 +++++++++++ 3 files changed, 23 insertions(+) diff --git a/dbtmetabase/__main__.py b/dbtmetabase/__main__.py index e225d9a2..9e69c85b 100644 --- a/dbtmetabase/__main__.py +++ b/dbtmetabase/__main__.py @@ -359,12 +359,20 @@ def models( is_flag=True, help="Include personal Metabase collections.", ) +@click.option( + "--exclude-unverified", + envvar="EXCLUDE_UNVERIFIED", + show_envvar=True, + is_flag=True, + help="Exclude collection items that have not been verified.", +) def exposures( output_path: str, output_grouping: Optional[str], include_collections: Optional[Sequence[str]], exclude_collections: Optional[Sequence[str]], allow_personal_collections: bool, + exclude_unverified: bool, core: DbtMetabase, ): core.extract_exposures( @@ -375,6 +383,7 @@ def exposures( exclude=exclude_collections, ), allow_personal_collections=allow_personal_collections, + exclude_unverified=exclude_unverified, ) diff --git a/dbtmetabase/_exposures.py b/dbtmetabase/_exposures.py index 35b95e96..ddace65a 100644 --- a/dbtmetabase/_exposures.py +++ b/dbtmetabase/_exposures.py @@ -46,6 +46,7 @@ def extract_exposures( output_grouping: Optional[str] = None, collection_filter: Optional[Filter] = None, allow_personal_collections: bool = False, + exclude_unverified: bool = False, ) -> Iterable[Mapping]: """Extract dbt exposures from Metabase. @@ -54,6 +55,7 @@ def extract_exposures( output_grouping (Optional[str], optional): Grouping for output YAML files, supported values: "collection" (by collection slug) or "type" (by entity type). Defaults to None. collection_filter (Optional[Filter], optional): Filter Metabase collections. Defaults to None. allow_personal_collections (bool, optional): Allow personal Metabase collections. Defaults to False. + exlcude_unverified (bool, optional): Exclude unverified models. Defaults to False. Returns: Iterable[Mapping]: List of parsed exposures. @@ -88,6 +90,7 @@ def extract_exposures( for item in self.metabase.get_collection_items( uid=collection["id"], models=("card", "dashboard"), + exclude_unverified=exclude_unverified, ): depends = set() native_query = "" diff --git a/dbtmetabase/metabase.py b/dbtmetabase/metabase.py index 5e66de95..0fa8fea3 100644 --- a/dbtmetabase/metabase.py +++ b/dbtmetabase/metabase.py @@ -131,6 +131,7 @@ def get_collection_items( self, uid: str, models: Sequence[str], + exclude_unverified: bool = False, ) -> Sequence[Mapping]: """Retrieves collection items of specific types (e.g. card, dashboard, collection).""" results = list( @@ -140,6 +141,16 @@ def get_collection_items( params={"models": models}, ) ) + if exclude_unverified: + verified_results = list( + self._api( + method="get", + path=f"/api/search/", + params={"models": models, "verified": True}, + ) + ) + verified_ids = [y['id'] for y in verified_results] + results = [x for x in results if x['id'] in verified_ids] results = list(filter(lambda x: x["model"] in models, results)) return results From 31a7f6aa54b0770e8de2b1c5d25f4ca106dd2f72 Mon Sep 17 00:00:00 2001 From: Alex Roy Date: Thu, 14 Mar 2024 13:58:45 -0400 Subject: [PATCH 2/6] simplify check for moderation status --- dbtmetabase/_exposures.py | 4 +++- dbtmetabase/metabase.py | 11 ----------- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/dbtmetabase/_exposures.py b/dbtmetabase/_exposures.py index ddace65a..f9c752a3 100644 --- a/dbtmetabase/_exposures.py +++ b/dbtmetabase/_exposures.py @@ -90,8 +90,10 @@ def extract_exposures( for item in self.metabase.get_collection_items( uid=collection["id"], models=("card", "dashboard"), - exclude_unverified=exclude_unverified, ): + if exclude_unverified and item.get("moderated_status") != "verified": + _logger.debug("Skipping unverified item '%s'", item["id"]) + continue depends = set() native_query = "" header = "" diff --git a/dbtmetabase/metabase.py b/dbtmetabase/metabase.py index 0fa8fea3..5e66de95 100644 --- a/dbtmetabase/metabase.py +++ b/dbtmetabase/metabase.py @@ -131,7 +131,6 @@ def get_collection_items( self, uid: str, models: Sequence[str], - exclude_unverified: bool = False, ) -> Sequence[Mapping]: """Retrieves collection items of specific types (e.g. card, dashboard, collection).""" results = list( @@ -141,16 +140,6 @@ def get_collection_items( params={"models": models}, ) ) - if exclude_unverified: - verified_results = list( - self._api( - method="get", - path=f"/api/search/", - params={"models": models, "verified": True}, - ) - ) - verified_ids = [y['id'] for y in verified_results] - results = [x for x in results if x['id'] in verified_ids] results = list(filter(lambda x: x["model"] in models, results)) return results From 03c51303edfe8cc0f46758fdc8c5550b7a52fb57 Mon Sep 17 00:00:00 2001 From: Alex Roy Date: Thu, 14 Mar 2024 14:14:23 -0400 Subject: [PATCH 3/6] leave dashboards alone --- dbtmetabase/__main__.py | 10 +++++----- dbtmetabase/_exposures.py | 12 ++++++++---- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/dbtmetabase/__main__.py b/dbtmetabase/__main__.py index 9e69c85b..aa506fe5 100644 --- a/dbtmetabase/__main__.py +++ b/dbtmetabase/__main__.py @@ -360,11 +360,11 @@ def models( help="Include personal Metabase collections.", ) @click.option( - "--exclude-unverified", - envvar="EXCLUDE_UNVERIFIED", + "--exclude-unverified-cards", + envvar="EXCLUDE_UNVERIFIED_CARDS", show_envvar=True, is_flag=True, - help="Exclude collection items that have not been verified.", + help="Exclude cards that have not been verified.", ) def exposures( output_path: str, @@ -372,7 +372,7 @@ def exposures( include_collections: Optional[Sequence[str]], exclude_collections: Optional[Sequence[str]], allow_personal_collections: bool, - exclude_unverified: bool, + exclude_unverified_cards: bool, core: DbtMetabase, ): core.extract_exposures( @@ -383,7 +383,7 @@ def exposures( exclude=exclude_collections, ), allow_personal_collections=allow_personal_collections, - exclude_unverified=exclude_unverified, + exclude_unverified_cards=exclude_unverified_cards, ) diff --git a/dbtmetabase/_exposures.py b/dbtmetabase/_exposures.py index f9c752a3..a784ffc4 100644 --- a/dbtmetabase/_exposures.py +++ b/dbtmetabase/_exposures.py @@ -46,7 +46,7 @@ def extract_exposures( output_grouping: Optional[str] = None, collection_filter: Optional[Filter] = None, allow_personal_collections: bool = False, - exclude_unverified: bool = False, + exclude_unverified_cards: bool = False, ) -> Iterable[Mapping]: """Extract dbt exposures from Metabase. @@ -55,7 +55,7 @@ def extract_exposures( output_grouping (Optional[str], optional): Grouping for output YAML files, supported values: "collection" (by collection slug) or "type" (by entity type). Defaults to None. collection_filter (Optional[Filter], optional): Filter Metabase collections. Defaults to None. allow_personal_collections (bool, optional): Allow personal Metabase collections. Defaults to False. - exlcude_unverified (bool, optional): Exclude unverified models. Defaults to False. + exclude_unverified_cards (bool, optional): Exclude unverified cards. Defaults to False. Returns: Iterable[Mapping]: List of parsed exposures. @@ -91,8 +91,12 @@ def extract_exposures( uid=collection["id"], models=("card", "dashboard"), ): - if exclude_unverified and item.get("moderated_status") != "verified": - _logger.debug("Skipping unverified item '%s'", item["id"]) + if ( + exclude_unverified_cards + and item.get("moderated_status") != "verified" + and item.get("model") == "card" + ): + _logger.debug("Skipping unverified card '%s'", item["name"]) continue depends = set() native_query = "" From 964ad98fe5975def5277d6c754bd5a43e62c9535 Mon Sep 17 00:00:00 2001 From: alex-float-on <159055527+alex-float-on@users.noreply.github.com> Date: Thu, 14 Mar 2024 21:04:53 -0400 Subject: [PATCH 4/6] Add Co-authored-by: Mike Gouline <1960272+gouline@users.noreply.github.com> --- dbtmetabase/_exposures.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dbtmetabase/_exposures.py b/dbtmetabase/_exposures.py index 292c4852..94ba2882 100644 --- a/dbtmetabase/_exposures.py +++ b/dbtmetabase/_exposures.py @@ -98,6 +98,7 @@ def extract_exposures( ): _logger.debug("Skipping unverified card '%s'", item["name"]) continue + depends = set() native_query = "" header = "" From da0ea82070111238f770c9e3e782f37c0548791e Mon Sep 17 00:00:00 2001 From: alex-float-on <159055527+alex-float-on@users.noreply.github.com> Date: Thu, 14 Mar 2024 21:14:32 -0400 Subject: [PATCH 5/6] Revise ordering of exclusion logic Co-authored-by: Mike Gouline <1960272+gouline@users.noreply.github.com> --- dbtmetabase/_exposures.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbtmetabase/_exposures.py b/dbtmetabase/_exposures.py index 94ba2882..25b2d5ad 100644 --- a/dbtmetabase/_exposures.py +++ b/dbtmetabase/_exposures.py @@ -93,8 +93,8 @@ def extract_exposures( ): if ( exclude_unverified_cards + and item["model"] == "card" and item.get("moderated_status") != "verified" - and item.get("model") == "card" ): _logger.debug("Skipping unverified card '%s'", item["name"]) continue From 894b497e7f1b8df7c922fd2694e4662e5b5d82fb Mon Sep 17 00:00:00 2001 From: Alex Roy Date: Thu, 14 Mar 2024 21:43:36 -0400 Subject: [PATCH 6/6] revert flag name now that we have alignment --- dbtmetabase/__main__.py | 10 +++++----- dbtmetabase/_exposures.py | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/dbtmetabase/__main__.py b/dbtmetabase/__main__.py index aa506fe5..779c5c5f 100644 --- a/dbtmetabase/__main__.py +++ b/dbtmetabase/__main__.py @@ -360,11 +360,11 @@ def models( help="Include personal Metabase collections.", ) @click.option( - "--exclude-unverified-cards", - envvar="EXCLUDE_UNVERIFIED_CARDS", + "--exclude-unverified", + envvar="EXCLUDE_UNVERIFIED", show_envvar=True, is_flag=True, - help="Exclude cards that have not been verified.", + help="Exclude items that have not been verified. Only applies to entity types that support verification.", ) def exposures( output_path: str, @@ -372,7 +372,7 @@ def exposures( include_collections: Optional[Sequence[str]], exclude_collections: Optional[Sequence[str]], allow_personal_collections: bool, - exclude_unverified_cards: bool, + exclude_unverified: bool, core: DbtMetabase, ): core.extract_exposures( @@ -383,7 +383,7 @@ def exposures( exclude=exclude_collections, ), allow_personal_collections=allow_personal_collections, - exclude_unverified_cards=exclude_unverified_cards, + exclude_unverified=exclude_unverified, ) diff --git a/dbtmetabase/_exposures.py b/dbtmetabase/_exposures.py index 25b2d5ad..b4cac4c4 100644 --- a/dbtmetabase/_exposures.py +++ b/dbtmetabase/_exposures.py @@ -46,7 +46,7 @@ def extract_exposures( output_grouping: Optional[str] = None, collection_filter: Optional[Filter] = None, allow_personal_collections: bool = False, - exclude_unverified_cards: bool = False, + exclude_unverified: bool = False, ) -> Iterable[Mapping]: """Extract dbt exposures from Metabase. @@ -55,7 +55,7 @@ def extract_exposures( output_grouping (Optional[str], optional): Grouping for output YAML files, supported values: "collection" (by collection slug) or "type" (by entity type). Defaults to None. collection_filter (Optional[Filter], optional): Filter Metabase collections. Defaults to None. allow_personal_collections (bool, optional): Allow personal Metabase collections. Defaults to False. - exclude_unverified_cards (bool, optional): Exclude unverified cards. Defaults to False. + exclude_unverified (bool, optional): Exclude items that have not been verified. Only applies to entity types that support verification. Defaults to False. Returns: Iterable[Mapping]: List of parsed exposures. @@ -92,7 +92,7 @@ def extract_exposures( models=("card", "dashboard"), ): if ( - exclude_unverified_cards + exclude_unverified and item["model"] == "card" and item.get("moderated_status") != "verified" ):