Skip to content

Commit

Permalink
Use daf_butler subclass for SIAv2Handler with entry point
Browse files Browse the repository at this point in the history
This provides a way for other butler universes to substitute
their own handlers.
  • Loading branch information
timj committed Jan 10, 2025
1 parent ebbe7df commit f38174a
Show file tree
Hide file tree
Showing 3 changed files with 182 additions and 70 deletions.
2 changes: 2 additions & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,8 @@ dynamic = ["version"]

[project.entry-points.'butler.cli']
dax_obscore = "lsst.dax.obscore.cli:get_cli_subcommands"
[project.entry-points.'dax_obscore.siav2']
daf_butler = "lsst.dax.obscore.siav2:get_daf_butler_siav2_handler"

[project.urls]
"Homepage" = "https://github.com/lsst-dm/dax_obscore"
Expand Down
57 changes: 57 additions & 0 deletions python/lsst/dax/obscore/plugins.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
# This file is part of dax_obscore.
#
# Developed for the LSST Data Management System.
# This product includes software developed by the LSST Project
# (http://www.lsst.org).
# See the COPYRIGHT file at the top-level directory of this distribution
# for details of code ownership.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.

from __future__ import annotations

from importlib.metadata import EntryPoint, entry_points
from typing import TYPE_CHECKING

if TYPE_CHECKING:
from .siav2 import SIAv2Handler

Check warning on line 28 in python/lsst/dax/obscore/plugins.py

View check run for this annotation

Codecov / codecov/patch

python/lsst/dax/obscore/plugins.py#L28

Added line #L28 was not covered by tests


def _get_siav2_entry_points() -> dict[str, EntryPoint]:
plugins = entry_points(group="dax_obscore.siav2")
return {p.name: p for p in plugins}


def get_siav2_handler(namespace: str) -> type[SIAv2Handler]:
"""Select the correct handler for this universe namespace.
Parameters
----------
namespace : `str`
The butler dimension universe namespace.
Returns
-------
handler : `type` [ `lsst.dax.obscore.siav2.SIAv2Handler` ]
The class of handler suitable for this namespace.
"""
plugins = _get_siav2_entry_points()
entry_point = plugins.get(namespace)
if entry_point is None:
known = ", ".join(plugins.keys())
raise RuntimeError(

Check warning on line 53 in python/lsst/dax/obscore/plugins.py

View check run for this annotation

Codecov / codecov/patch

python/lsst/dax/obscore/plugins.py#L52-L53

Added lines #L52 - L53 were not covered by tests
f"Unable to find suitable SIAv2 Handler for namespace {namespace} [do understand: {known}]"
)
func = entry_point.load()
return func()
193 changes: 123 additions & 70 deletions python/lsst/dax/obscore/siav2.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@

import math
import numbers
from abc import abstractmethod
from collections import defaultdict
from collections.abc import Iterable, Iterator, Sequence
from typing import Any, Self
Expand All @@ -40,6 +41,7 @@

from .config import ExporterConfig, WhereBind
from .obscore_exporter import ObscoreExporter
from .plugins import get_siav2_handler

_LOG = getLogger(__name__)
_VALID_CALIB = frozenset({0, 1, 2, 3})
Expand Down Expand Up @@ -241,6 +243,7 @@ def __init__(self, butler: Butler, config: ExporterConfig):
self.config = config
self.warnings: list[str] = []

@abstractmethod
def get_all_instruments(self) -> list[str]:
"""Query butler for all known instruments.
Expand All @@ -249,8 +252,9 @@ def get_all_instruments(self) -> list[str]:
instruments : `list` [ `str` ]
All the instrument names known to this butler.
"""
return [rec.name for rec in self.butler.query_dimension_records("instrument")]
raise NotImplementedError()

Check warning on line 255 in python/lsst/dax/obscore/siav2.py

View check run for this annotation

Codecov / codecov/patch

python/lsst/dax/obscore/siav2.py#L255

Added line #L255 was not covered by tests

@abstractmethod
def get_band_information(
self, instruments: list[str], band_intervals: Iterable[Interval]
) -> dict[str, Any]:
Expand Down Expand Up @@ -279,27 +283,7 @@ def get_band_information(
``bands`` key is a set of bands associated with those filters
independent of instrument.
"""
matching_filters = defaultdict(set)
matching_bands = set()
with self.butler.query() as query:
records = query.dimension_records("physical_filter")
if instruments:
records = records.where("instrument in (INSTRUMENTS)", bind={"INSTRUMENTS": instruments})
for rec in records:
if (spec_range := self.config.spectral_ranges.get(rec.name)) is not None:
spec_interval = Interval(start=spec_range[0], end=spec_range[1])
assert spec_range[0] is not None # for mypy
assert spec_range[1] is not None
for band_interval in band_intervals:
if band_interval.overlaps(spec_interval):
matching_filters[rec.instrument].add(rec.name)
matching_bands.add(rec.band)
else:
self.warnings.append(
f"Ignoring physical filter {rec.name} from instrument {rec.instrument}"
" since it has no defined spectral range"
)
return {"filters": matching_filters, "bands": matching_bands}
raise NotImplementedError()

Check warning on line 286 in python/lsst/dax/obscore/siav2.py

View check run for this annotation

Codecov / codecov/patch

python/lsst/dax/obscore/siav2.py#L286

Added line #L286 was not covered by tests

def process_query(self, parameters: SIAv2Parameters) -> dict[str, list[WhereBind]]:
"""Process an SIAv2 query.
Expand Down Expand Up @@ -386,6 +370,7 @@ def process_query(self, parameters: SIAv2Parameters) -> dict[str, list[WhereBind

return dataset_type_wheres

@abstractmethod
def from_instrument_or_band(
self, instruments: list[str], band_info: dict[str, Any], dimensions: DimensionGroup
) -> tuple[list[WhereBind], WhereBind | None]:
Expand All @@ -410,6 +395,108 @@ def from_instrument_or_band(
general_wheres : `WhereBind` | None
Query constraint that is not instrument-specific.
"""
raise NotImplementedError()

Check warning on line 398 in python/lsst/dax/obscore/siav2.py

View check run for this annotation

Codecov / codecov/patch

python/lsst/dax/obscore/siav2.py#L398

Added line #L398 was not covered by tests

@abstractmethod
def from_pos(self, regions: Sequence[Region], dimensions: DimensionGroup) -> WhereBind | None:
"""Convert a region request to a butler where clause.
Parameters
----------
regions : `~collections.abc.Iterable` [ `lsst.sphgeom.Region` ]
The region of interest.
dimensions : `lsst.daf.butler.DimensionGroup`
The dimensions for the dataset type being queried.
Returns
-------
where : `WhereBind` or `None`
The where clause or `None` if region is not supported by this
dataset type.
"""
raise NotImplementedError()

Check warning on line 417 in python/lsst/dax/obscore/siav2.py

View check run for this annotation

Codecov / codecov/patch

python/lsst/dax/obscore/siav2.py#L417

Added line #L417 was not covered by tests

@abstractmethod
def from_time(
self, ts: Iterable[astropy.time.Time | Timespan], dimensions: DimensionGroup
) -> WhereBind | None:
"""Convert a time span to a butler where clause.
Parameters
----------
ts : `~collections.abc.Iterable` of \
[ `astropy.time.Time` | `lsst.daf.butler.Timespan` ]
The times of interest.
dimensions : `lsst.daf.butler.DimensionGroup`
The dimensions for the dataset type being queried.
Returns
-------
where : `WhereBind` or `None`
The where clause or `None` if time is not supported by this
dataset type.
"""
raise NotImplementedError()

Check warning on line 439 in python/lsst/dax/obscore/siav2.py

View check run for this annotation

Codecov / codecov/patch

python/lsst/dax/obscore/siav2.py#L439

Added line #L439 was not covered by tests

@abstractmethod
def from_exptime(
self, exptime_intervals: Iterable[Interval], dimensions: DimensionGroup
) -> WhereBind | None:
"""Convert an exposure time interval to a butler where clause.
Parameters
----------
exptime_intervals : `~collections.abc.Iterable` [ `Interval` ]
The exposure time interval of interest as two floating point UTC
MJDs.
dimensions : `lsst.daf.butler.DimensionGroup`
The dimensions for the dataset type being queried.
Returns
-------
wheres : `WhereBind` or `None`
The where clause for the exposure times, or `None` if the dataset
type does not understand exposure time.
"""
raise NotImplementedError()

Check warning on line 461 in python/lsst/dax/obscore/siav2.py

View check run for this annotation

Codecov / codecov/patch

python/lsst/dax/obscore/siav2.py#L461

Added line #L461 was not covered by tests


class SIAv2DafButlerHandler(SIAv2Handler):
"""Process SIAv2 query parameters using the daf_butler dimension universe
and convert into Butler query constraints.
"""

def get_all_instruments(self) -> list[str]:
return [rec.name for rec in self.butler.query_dimension_records("instrument")]

def get_band_information(
self, instruments: list[str], band_intervals: Iterable[Interval]
) -> dict[str, Any]:
matching_filters = defaultdict(set)
matching_bands = set()
with self.butler.query() as query:
records = query.dimension_records("physical_filter")
if instruments:
records = records.where("instrument in (INSTRUMENTS)", bind={"INSTRUMENTS": instruments})
for rec in records:
if (spec_range := self.config.spectral_ranges.get(rec.name)) is not None:
spec_interval = Interval(start=spec_range[0], end=spec_range[1])
assert spec_range[0] is not None # for mypy
assert spec_range[1] is not None
for band_interval in band_intervals:
if band_interval.overlaps(spec_interval):
matching_filters[rec.instrument].add(rec.name)
matching_bands.add(rec.band)
else:
self.warnings.append(

Check warning on line 491 in python/lsst/dax/obscore/siav2.py

View check run for this annotation

Codecov / codecov/patch

python/lsst/dax/obscore/siav2.py#L491

Added line #L491 was not covered by tests
f"Ignoring physical filter {rec.name} from instrument {rec.instrument}"
" since it has no defined spectral range"
)
return {"filters": matching_filters, "bands": matching_bands}

def from_instrument_or_band(
self, instruments: list[str], band_info: dict[str, Any], dimensions: DimensionGroup
) -> tuple[list[WhereBind], WhereBind | None]:
instrument_wheres = []
general_where: WhereBind | None = None
if "instrument" in dimensions and instruments:
Expand All @@ -435,21 +522,6 @@ def from_instrument_or_band(
return instrument_wheres, general_where

def from_pos(self, regions: Sequence[Region], dimensions: DimensionGroup) -> WhereBind | None:
"""Convert a region request to a butler where clause.
Parameters
----------
regions : `~collections.abc.Iterable` [ `lsst.sphgeom.Region` ]
The region of interest.
dimensions : `lsst.daf.butler.DimensionGroup`
The dimensions for the dataset type being queried.
Returns
-------
where : `WhereBind` or `None`
The where clause or `None` if region is not supported by this
dataset type.
"""
extra_dims = set()
region_dim = dimensions.region_dimension
if not region_dim:
Expand Down Expand Up @@ -479,22 +551,6 @@ def from_pos(self, regions: Sequence[Region], dimensions: DimensionGroup) -> Whe
def from_time(
self, ts: Iterable[astropy.time.Time | Timespan], dimensions: DimensionGroup
) -> WhereBind | None:
"""Convert a time span to a butler where clause.
Parameters
----------
ts : `~collections.abc.Iterable` of \
[ `astropy.time.Time` | `lsst.daf.butler.Timespan` ]
The times of interest.
dimensions : `lsst.daf.butler.DimensionGroup`
The dimensions for the dataset type being queried.
Returns
-------
where : `WhereBind` or `None`
The where clause or `None` if time is not supported by this
dataset type.
"""
time_dim = dimensions.timespan_dimension
if not time_dim:
return None
Expand All @@ -507,22 +563,6 @@ def from_time(
def from_exptime(
self, exptime_intervals: Iterable[Interval], dimensions: DimensionGroup
) -> WhereBind | None:
"""Convert an exposure time interval to a butler where clause.
Parameters
----------
exptime_intervals : `~collections.abc.Iterable` [ `Interval` ]
The exposure time interval of interest as two floating point UTC
MJDs.
dimensions : `lsst.daf.butler.DimensionGroup`
The dimensions for the dataset type being queried.
Returns
-------
wheres : `WhereBind` or `None`
The where clause for the exposure times, or `None` if the dataset
type does not understand exposure time.
"""
if "exposure" in dimensions:
exp_dim = "exposure"
elif "visit" in dimensions:
Expand All @@ -543,6 +583,18 @@ def from_exptime(
return WhereBind.combine(wheres, mode="OR")


def get_daf_butler_siav2_handler() -> type[SIAv2Handler]:
"""Return the SIAv2 handler specifically designed for the daf_butler
namespace.
Returns
-------
handler : `type` [ `lsst.dax.obscore.siav2.SIAv2Handler` ]
The handler to be used for daf_butler universes.
"""
return SIAv2DafButlerHandler


def siav2_query_from_raw(
butler: Butler,
config: ExporterConfig,
Expand Down Expand Up @@ -686,7 +738,8 @@ def siav2_query(

_LOG.verbose("Received parameters: %s", parameters)

handler = SIAv2Handler(butler, cfg)
handler_type = get_siav2_handler(butler.dimensions.namespace)
handler = handler_type(butler, cfg)
cfg.dataset_type_constraints = handler.process_query(parameters)

# Downselect the dataset types being queried -- a missing constraint
Expand Down

0 comments on commit f38174a

Please sign in to comment.