From 165cdd2c04be460ad09f20b73f21842eb9342f6d Mon Sep 17 00:00:00 2001 From: "David H. Irving" Date: Fri, 22 Nov 2024 14:04:02 -0700 Subject: [PATCH 1/2] Add --chains NO-CHILDREN to query-collections Added a `--chains NO-CHILDREN` mode to the `butler query-collections` CLI, which returns results without recursing into `CHAINED` collections. --- doc/changes/DM-47768.feature.md | 2 ++ python/lsst/daf/butler/cli/cmd/commands.py | 5 ++++- .../daf/butler/script/queryCollections.py | 20 ++++++++++--------- tests/test_cliCmdQueryCollections.py | 18 +++++++++++++++++ 4 files changed, 35 insertions(+), 10 deletions(-) create mode 100644 doc/changes/DM-47768.feature.md diff --git a/doc/changes/DM-47768.feature.md b/doc/changes/DM-47768.feature.md new file mode 100644 index 0000000000..c62fd45df0 --- /dev/null +++ b/doc/changes/DM-47768.feature.md @@ -0,0 +1,2 @@ +Added a `--chains NO-CHILDREN` mode to the `butler query-collections` CLI, +which returns results without recursing into `CHAINED` collections. diff --git a/python/lsst/daf/butler/cli/cmd/commands.py b/python/lsst/daf/butler/cli/cmd/commands.py index 8b55614115..782b6187fb 100644 --- a/python/lsst/daf/butler/cli/cmd/commands.py +++ b/python/lsst/daf/butler/cli/cmd/commands.py @@ -409,13 +409,16 @@ def prune_datasets(**kwargs: Any) -> None: FLATTEN lists all datasets, including child datasets, in one list. + NO-CHILDREN lists all collections in one list. CHAINED collections are + included, but they are not expanded to include their children. + [default: TREE]""", # above, the default value is included, instead of using show_default, so # that the default is printed on its own line instead of coming right after # the FLATTEN text. callback=to_upper, type=click.Choice( - choices=("TABLE", "INVERSE-TABLE", "TREE", "INVERSE-TREE", "FLATTEN"), + choices=("TABLE", "INVERSE-TABLE", "TREE", "INVERSE-TREE", "FLATTEN", "NO-CHILDREN"), case_sensitive=False, ), ) diff --git a/python/lsst/daf/butler/script/queryCollections.py b/python/lsst/daf/butler/script/queryCollections.py index f699df4c25..341f461312 100644 --- a/python/lsst/daf/butler/script/queryCollections.py +++ b/python/lsst/daf/butler/script/queryCollections.py @@ -28,6 +28,7 @@ from __future__ import annotations from collections.abc import Iterable +from typing import Literal from astropy.table import Table @@ -166,15 +167,16 @@ def addCollection(info: CollectionInfo, level: int = 0) -> None: return table -def _getFlatten( - repo: str, - glob: Iterable[str], - collection_type: Iterable[CollectionType], +def _getList( + repo: str, glob: Iterable[str], collection_type: Iterable[CollectionType], flatten_chains: bool ) -> Table: + """Return collection results as a table representing a flat list of + collections. + """ butler = Butler.from_config(repo) collections = list( butler.collections.query_info( - glob or "*", collection_types=frozenset(collection_type), flatten_chains=True + glob or "*", collection_types=frozenset(collection_type), flatten_chains=flatten_chains ) ) names = [c.name for c in collections] @@ -186,7 +188,7 @@ def queryCollections( repo: str, glob: Iterable[str], collection_type: Iterable[CollectionType], - chains: str, + chains: Literal["INVERSE-TABLE", "TABLE", "TREE", "INVERSE-TREE", "FLATTEN", "NO-CHILDREN"], ) -> Table: """Get the collections whose names match an expression. @@ -202,7 +204,6 @@ def queryCollections( optional If provided, only return collections of these types. chains : `str` - Must be one of "FLATTEN", "TABLE", or "TREE" (case sensitive). Affects contents and formatting of results, see ``cli.commands.query_collections``. @@ -215,6 +216,7 @@ def queryCollections( return _getTable(repo, glob, collection_type, inverse) elif (inverse := chains == "INVERSE-TREE") or chains == "TREE": return _getTree(repo, glob, collection_type, inverse) - elif chains == "FLATTEN": - return _getFlatten(repo, glob, collection_type) + elif chains == "FLATTEN" or chains == "NO-CHILDREN": + flatten = chains == "FLATTEN" + return _getList(repo, glob, collection_type, flatten) raise RuntimeError(f"Value for --chains not recognized: {chains}") diff --git a/tests/test_cliCmdQueryCollections.py b/tests/test_cliCmdQueryCollections.py index 4427fe0f41..23d04b9439 100644 --- a/tests/test_cliCmdQueryCollections.py +++ b/tests/test_cliCmdQueryCollections.py @@ -277,6 +277,24 @@ def testChained(self): ) self.assertAstropyTablesEqual(readTable(result.output), expected, unorderedRows=True) + result = self.runner.invoke(cli, ["query-collections", "here", "--chains", "NO-CHILDREN"]) + self.assertEqual(result.exit_code, 0, clickResultMsg(result)) + expected = Table( + array( + ( + ("calibration1", "CALIBRATION"), + ("chain1", "CHAINED"), + ("chain2", "CHAINED"), + ("imported_g", "RUN"), + ("imported_r", "RUN"), + ("run1", "RUN"), + ("tag1", "TAGGED"), + ) + ), + names=("Name", "Type"), + ) + self.assertAstropyTablesEqual(readTable(result.output), expected, unorderedRows=True) + # Add a couple more run collections for chain testing registry1.registerRun("run2") registry1.registerRun("run3") From 3e6aac1cbaddf47e239349fa7dec0982fdb6c42c Mon Sep 17 00:00:00 2001 From: "David H. Irving" Date: Fri, 22 Nov 2024 14:06:54 -0700 Subject: [PATCH 2/2] Tweak --chains doc Replace "datasets" with "collections" throughout, since this command has nothing to do with datasets. --- python/lsst/daf/butler/cli/cmd/commands.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/python/lsst/daf/butler/cli/cmd/commands.py b/python/lsst/daf/butler/cli/cmd/commands.py index 782b6187fb..86e8a8e36f 100644 --- a/python/lsst/daf/butler/cli/cmd/commands.py +++ b/python/lsst/daf/butler/cli/cmd/commands.py @@ -394,20 +394,22 @@ def prune_datasets(**kwargs: Any) -> None: default="TREE", help="""Affects how results are presented: - TABLE lists each dataset in table form, with columns for dataset name - and type, and a column that lists children of CHAINED datasets (if any - CHAINED datasets are found). + TABLE lists each collection in table form, with columns for collection + name and type, and a column that lists children of CHAINED collections + (if any CHAINED collections are found). INVERSE-TABLE is like TABLE but instead of a column listing CHAINED - dataset children, it lists the parents of the dataset if it is contained - in any CHAINED collections. + collection children, it lists the parents of the collection if it is + contained in any CHAINED collections. - TREE recursively lists children below each CHAINED dataset in tree form. + TREE recursively lists children below each CHAINED collection in tree + form. - INVERSE-TREE recursively lists parent datasets below each dataset in - tree form. + INVERSE-TREE recursively lists parent collections below each collection + in tree form. - FLATTEN lists all datasets, including child datasets, in one list. + FLATTEN lists all collections, including children of CHAINED + collections, in one list. NO-CHILDREN lists all collections in one list. CHAINED collections are included, but they are not expanded to include their children.