diff --git a/python/lsst/daf/butler/remote_butler/__init__.py b/python/lsst/daf/butler/remote_butler/__init__.py new file mode 100644 index 0000000000..98bb440c81 --- /dev/null +++ b/python/lsst/daf/butler/remote_butler/__init__.py @@ -0,0 +1,28 @@ +# This file is part of daf_butler. +# +# Developed for the LSST Data Management System. +# This product includes software developed by the LSST Project +# (http://www.lsst.org). +# See the COPYRIGHT file at the top-level directory of this distribution +# for details of code ownership. +# +# This software is dual licensed under the GNU General Public License and also +# under a 3-clause BSD license. Recipients may choose which of these licenses +# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, +# respectively. If you choose the GPL option then the following text applies +# (but note that there is still no warranty even if you opt for BSD instead): +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from ._remote_butler import RemoteButler diff --git a/python/lsst/daf/butler/remote_butler/_config.py b/python/lsst/daf/butler/remote_butler/_config.py new file mode 100644 index 0000000000..cdb41d4a4d --- /dev/null +++ b/python/lsst/daf/butler/remote_butler/_config.py @@ -0,0 +1,37 @@ +# This file is part of daf_butler. +# +# Developed for the LSST Data Management System. +# This product includes software developed by the LSST Project +# (http://www.lsst.org). +# See the COPYRIGHT file at the top-level directory of this distribution +# for details of code ownership. +# +# This software is dual licensed under the GNU General Public License and also +# under a 3-clause BSD license. Recipients may choose which of these licenses +# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, +# respectively. If you choose the GPL option then the following text applies +# (but note that there is still no warranty even if you opt for BSD instead): +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from pydantic import AnyHttpUrl +from .._compat import _BaseModelCompat + + +class RemoteButlerOptionsModel(_BaseModelCompat): + url: AnyHttpUrl + + +class RemoteButlerConfigModel(_BaseModelCompat): + remote_butler: RemoteButlerOptionsModel diff --git a/python/lsst/daf/butler/remote_butler/_remote_butler.py b/python/lsst/daf/butler/remote_butler/_remote_butler.py new file mode 100644 index 0000000000..ffd792dcc5 --- /dev/null +++ b/python/lsst/daf/butler/remote_butler/_remote_butler.py @@ -0,0 +1,261 @@ +# This file is part of daf_butler. +# +# Developed for the LSST Data Management System. +# This product includes software developed by the LSST Project +# (http://www.lsst.org). +# See the COPYRIGHT file at the top-level directory of this distribution +# for details of code ownership. +# +# This software is dual licensed under the GNU General Public License and also +# under a 3-clause BSD license. Recipients may choose which of these licenses +# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, +# respectively. If you choose the GPL option then the following text applies +# (but note that there is still no warranty even if you opt for BSD instead): +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from collections.abc import Collection, Iterable, Sequence +from contextlib import AbstractContextManager +from typing import Any, TextIO + +from lsst.resources import ResourcePath, ResourcePathExpression + +from .._butler import Butler +from .._butler_config import ButlerConfig +from .._config import Config +from .._dataset_existence import DatasetExistence +from .._dataset_ref import DatasetIdGenEnum, DatasetRef +from .._dataset_type import DatasetType +from .._deferredDatasetHandle import DeferredDatasetHandle +from .._file_dataset import FileDataset +from .._limited_butler import LimitedButler +from .._storage_class import StorageClass +from ..datastore import DatasetRefURIs +from ..dimensions import DataId, DimensionUniverse +from ..registry import Registry +from ..transfers import RepoExportContext +from ._config import RemoteButlerConfigModel + + +class RemoteButler(Butler): + def __init__( + self, + config: Config | ResourcePathExpression | None = None, + searchPaths: Sequence[ResourcePathExpression] | None = None, + **kwargs: Any, + ): + butler_config = ButlerConfig(config, searchPaths, without_datastore=True) + self._config = RemoteButlerConfigModel.model_validate(butler_config) + + def isWriteable(self) -> bool: + # Docstring inherited. + return False + + @property + def dimensions(self) -> DimensionUniverse: + # Docstring inherited. + raise NotImplementedError() + + def transaction(self) -> AbstractContextManager[None]: + """This method will always raise NotImplementedError -- transactions are not supported by RemoteButler.""" + raise NotImplementedError() + + def put( + self, + obj: Any, + datasetRefOrType: DatasetRef | DatasetType | str, + /, + dataId: DataId | None = None, + *, + run: str | None = None, + **kwargs: Any, + ) -> DatasetRef: + # Docstring inherited. + raise NotImplementedError() + + def getDeferred( + self, + datasetRefOrType: DatasetRef | DatasetType | str, + /, + dataId: DataId | None = None, + *, + parameters: dict | None = None, + collections: Any = None, + storageClass: str | StorageClass | None = None, + **kwargs: Any, + ) -> DeferredDatasetHandle: + # Docstring inherited. + raise NotImplementedError() + + def get( + self, + datasetRefOrType: DatasetRef | DatasetType | str, + /, + dataId: DataId | None = None, + *, + parameters: dict[str, Any] | None = None, + collections: Any = None, + storageClass: StorageClass | str | None = None, + **kwargs: Any, + ) -> Any: + # Docstring inherited. + raise NotImplementedError() + + def getURIs( + self, + datasetRefOrType: DatasetRef | DatasetType | str, + /, + dataId: DataId | None = None, + *, + predict: bool = False, + collections: Any = None, + run: str | None = None, + **kwargs: Any, + ) -> DatasetRefURIs: + # Docstring inherited. + raise NotImplementedError() + + def getURI( + self, + datasetRefOrType: DatasetRef | DatasetType | str, + /, + dataId: DataId | None = None, + *, + predict: bool = False, + collections: Any = None, + run: str | None = None, + **kwargs: Any, + ) -> ResourcePath: + # Docstring inherited. + raise NotImplementedError() + + def retrieveArtifacts( + self, + refs: Iterable[DatasetRef], + destination: ResourcePathExpression, + transfer: str = "auto", + preserve_path: bool = True, + overwrite: bool = False, + ) -> list[ResourcePath]: + # Docstring inherited. + raise NotImplementedError() + + def exists( + self, + dataset_ref_or_type: DatasetRef | DatasetType | str, + /, + data_id: DataId | None = None, + *, + full_check: bool = True, + collections: Any = None, + **kwargs: Any, + ) -> DatasetExistence: + # Docstring inherited. + raise NotImplementedError() + + def _exists_many( + self, + refs: Iterable[DatasetRef], + /, + *, + full_check: bool = True, + ) -> dict[DatasetRef, DatasetExistence]: + # Docstring inherited. + raise NotImplementedError() + + def removeRuns(self, names: Iterable[str], unstore: bool = True) -> None: + # Docstring inherited. + raise NotImplementedError() + + def ingest( + self, + *datasets: FileDataset, + transfer: str | None = "auto", + run: str | None = None, + idGenerationMode: DatasetIdGenEnum | None = None, + record_validation_info: bool = True, + ) -> None: + # Docstring inherited. + raise NotImplementedError() + + def export( + self, + *, + directory: str | None = None, + filename: str | None = None, + format: str | None = None, + transfer: str | None = None, + ) -> AbstractContextManager[RepoExportContext]: + # Docstring inherited. + raise NotImplementedError() + + def import_( + self, + *, + directory: ResourcePathExpression | None = None, + filename: ResourcePathExpression | TextIO | None = None, + format: str | None = None, + transfer: str | None = None, + skip_dimensions: set | None = None, + ) -> None: + # Docstring inherited. + raise NotImplementedError() + + def transfer_from( + self, + source_butler: LimitedButler, + source_refs: Iterable[DatasetRef], + transfer: str = "auto", + skip_missing: bool = True, + register_dataset_types: bool = False, + transfer_dimensions: bool = False, + ) -> Collection[DatasetRef]: + # Docstring inherited. + raise NotImplementedError() + + def validateConfiguration( + self, + logFailures: bool = False, + datasetTypeNames: Iterable[str] | None = None, + ignore: Iterable[str] | None = None, + ) -> None: + # Docstring inherited. + raise NotImplementedError() + + @property + def collections(self) -> Sequence[str]: + # Docstring inherited. + raise NotImplementedError() + + @property + def run(self) -> str | None: + # Docstring inherited. + raise NotImplementedError() + + @property + def registry(self) -> Registry: + # Docstring inherited. + raise NotImplementedError() + + def pruneDatasets( + self, + refs: Iterable[DatasetRef], + *, + disassociate: bool = True, + unstore: bool = False, + tags: Iterable[str] = (), + purge: bool = False, + ) -> None: + # Docstring inherited. + raise NotImplementedError() diff --git a/tests/test_remote_butler.py b/tests/test_remote_butler.py new file mode 100644 index 0000000000..24ae1fb697 --- /dev/null +++ b/tests/test_remote_butler.py @@ -0,0 +1,47 @@ +# This file is part of daf_butler. +# +# Developed for the LSST Data Management System. +# This product includes software developed by the LSST Project +# (http://www.lsst.org). +# See the COPYRIGHT file at the top-level directory of this distribution +# for details of code ownership. +# +# This software is dual licensed under the GNU General Public License and also +# under a 3-clause BSD license. Recipients may choose which of these licenses +# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, +# respectively. If you choose the GPL option then the following text applies +# (but note that there is still no warranty even if you opt for BSD instead): +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import unittest + +from lsst.daf.butler import Butler +from lsst.daf.butler.remote_butler import RemoteButler +from pydantic import ValidationError + + +class RemoteButlerConfigTests(unittest.TestCase): + def test_instantiate_via_butler(self): + butler = Butler( + { + "cls": "lsst.daf.butler.remote_butler.RemoteButler", + "remote_butler": {"url": "https://validurl.example"}, + } + ) + assert isinstance(butler, RemoteButler) + + def test_bad_config(self): + with self.assertRaises(ValidationError): + Butler({"cls": "lsst.daf.butler.remote_butler.RemoteButler", "remote_butler": {"url": "!"}})