Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

DM-47375: Run query_all_datasets in a single request for RemoteButler #1114

Merged
merged 12 commits into from
Nov 14, 2024
Prev Previous commit
Next Next commit
Restrict --order-by in query-datasets to single type
The backend for querying multiple dataset types will not support "order by", so restrict the CLI to match the implementation.
dhirving committed Nov 14, 2024
commit 8da61ab2738209bff31ce0a163bfc65fa7c187c7
4 changes: 4 additions & 0 deletions python/lsst/daf/butler/script/queryDatasets.py
Original file line number Diff line number Diff line change
@@ -40,6 +40,7 @@
from .._exceptions import MissingDatasetTypeError
from .._query_all_datasets import DatasetsPage, query_all_datasets
from ..cli.utils import sortAstropyTable
from ..utils import has_globs

if TYPE_CHECKING:
from lsst.daf.butler import DatasetRef
@@ -205,6 +206,9 @@ def __init__(
)
glob = ["*"]

if order_by and (len(glob) > 1 or has_globs(glob)):
raise NotImplementedError("--order-by is only supported for queries with a single dataset type.")

# show_uri requires a datastore.
without_datastore = not show_uri
self.butler = butler or Butler.from_config(repo, without_datastore=without_datastore)
34 changes: 31 additions & 3 deletions tests/test_cliCmdQueryDatasets.py
Original file line number Diff line number Diff line change
@@ -344,7 +344,11 @@ def test_limit_order(self):

with self.assertLogs("lsst.daf.butler.script.queryDatasets", level="WARNING") as cm:
tables = self._queryDatasets(
repo=testRepo.butler, limit=-1, order_by=("visit",), collections="*", glob="*"
repo=testRepo.butler,
limit=-1,
order_by=("visit",),
collections="*",
glob=("test_metric_comp",),
)

self.assertIn("increase this limit", cm.output[0])
@@ -361,13 +365,21 @@ def test_limit_order(self):
# issued.
with self.assertNoLogs("lsst.daf.butler.script.queryDatasets", level="WARNING"):
tables = self._queryDatasets(
repo=testRepo.butler, limit=1, order_by=("visit",), collections="*", glob="*"
repo=testRepo.butler,
limit=1,
order_by=("visit",),
collections="*",
glob=("test_metric_comp",),
)
self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True)

with self.assertLogs("lsst.daf.butler.script.queryDatasets", level="WARNING") as cm:
tables = self._queryDatasets(
repo=testRepo.butler, limit=-1, order_by=("-visit",), collections="*", glob="*"
repo=testRepo.butler,
limit=-1,
order_by=("-visit",),
collections="*",
glob=("test_metric_comp",),
)
self.assertIn("increase this limit", cm.output[0])

@@ -379,6 +391,22 @@ def test_limit_order(self):
]
self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True)

# --order-by is not supported by the query backend for multiple dataset
# types, so we can only provide it for queries with a single dataset
# type.
with self.assertRaisesRegex(NotImplementedError, "--order-by"):
self._queryDatasets(
repo=testRepo.butler, limit=1, order_by=("visit",), collections="*", glob=["*"]
)
with self.assertRaisesRegex(NotImplementedError, "--order-by"):
self._queryDatasets(
repo=testRepo.butler,
limit=1,
order_by=("visit",),
collections="*",
glob=["test_metric_comp", "raw"],
)

def testFindFirstAndCollections(self):
"""Test the find-first option, and the collections option, since it
is required for find-first.