From 652de136b64802c94a7f7e9931ed091b7732fa6f Mon Sep 17 00:00:00 2001 From: Jakub Wlodek Date: Wed, 4 Sep 2024 13:16:59 -0400 Subject: [PATCH 01/60] Starting to work on ad tiff writer --- src/ophyd_async/epics/adcore/_core_io.py | 22 +-- src/ophyd_async/epics/adcore/_core_writer.py | 147 +++++++++++++++++++ src/ophyd_async/epics/adcore/_hdf_writer.py | 18 +-- src/ophyd_async/epics/adcore/_tiff_writer.py | 63 ++++++++ 4 files changed, 228 insertions(+), 22 deletions(-) create mode 100644 src/ophyd_async/epics/adcore/_core_writer.py create mode 100644 src/ophyd_async/epics/adcore/_tiff_writer.py diff --git a/src/ophyd_async/epics/adcore/_core_io.py b/src/ophyd_async/epics/adcore/_core_io.py index f15d48cd2e..e379623070 100644 --- a/src/ophyd_async/epics/adcore/_core_io.py +++ b/src/ophyd_async/epics/adcore/_core_io.py @@ -111,12 +111,8 @@ class Compression(str, Enum): jpeg = "JPEG" -class NDFileHDFIO(NDPluginBaseIO): +class NDFileIO(NDPluginBaseIO): def __init__(self, prefix: str, name="") -> None: - # Define some signals - self.position_mode = epics_signal_rw_rbv(bool, prefix + "PositionMode") - self.compression = epics_signal_rw_rbv(Compression, prefix + "Compression") - self.num_extra_dims = epics_signal_rw_rbv(int, prefix + "NumExtraDims") self.file_path = epics_signal_rw_rbv(str, prefix + "FilePath") self.file_name = epics_signal_rw_rbv(str, prefix + "FileName") self.file_path_exists = epics_signal_r(bool, prefix + "FilePathExists_RBV") @@ -127,12 +123,20 @@ def __init__(self, prefix: str, name="") -> None: ) self.num_capture = epics_signal_rw_rbv(int, prefix + "NumCapture") self.num_captured = epics_signal_r(int, prefix + "NumCaptured_RBV") - self.swmr_mode = epics_signal_rw_rbv(bool, prefix + "SWMRMode") - self.lazy_open = epics_signal_rw_rbv(bool, prefix + "LazyOpen") self.capture = epics_signal_rw_rbv(bool, prefix + "Capture") - self.flush_now = epics_signal_rw(bool, prefix + "FlushNow") - self.xml_file_name = epics_signal_rw_rbv(str, prefix + "XMLFileName") self.array_size0 = epics_signal_r(int, prefix + "ArraySize0") self.array_size1 = epics_signal_r(int, prefix + "ArraySize1") self.create_directory = epics_signal_rw(int, prefix + "CreateDirectory") + self.lazy_open = epics_signal_rw_rbv(bool, prefix + "LazyOpen") + super().__init__(prefix, name) + + +class NDFileHDFIO(NDFileIO): + def __init__(self, prefix: str, name="") -> None: + self.position_mode = epics_signal_rw_rbv(bool, prefix + "PositionMode") + self.compression = epics_signal_rw_rbv(Compression, prefix + "Compression") + self.num_extra_dims = epics_signal_rw_rbv(int, prefix + "NumExtraDims") + self.swmr_mode = epics_signal_rw_rbv(bool, prefix + "SWMRMode") + self.flush_now = epics_signal_rw(bool, prefix + "FlushNow") + self.xml_file_name = epics_signal_rw_rbv(str, prefix + "XMLFileName") super().__init__(prefix, name) diff --git a/src/ophyd_async/epics/adcore/_core_writer.py b/src/ophyd_async/epics/adcore/_core_writer.py new file mode 100644 index 0000000000..385c3032bb --- /dev/null +++ b/src/ophyd_async/epics/adcore/_core_writer.py @@ -0,0 +1,147 @@ + + +from pathlib import Path +from typing import AsyncGenerator, AsyncIterator, Optional +from ophyd_async.core._detector import DetectorWriter +from ophyd_async.core._providers import NameProvider, PathProvider, ShapeProvider +from ophyd_async.core._signal import observe_value, wait_for_value +from ophyd_async.core._status import AsyncStatus +from ophyd_async.core._utils import DEFAULT_TIMEOUT +from ._core_io import NDArrayBaseIO, NDFileIO + + +class ADWriter(DetectorWriter): + def __init__( + self, + fileio: NDFileIO, + path_provider: PathProvider, + name_provider: NameProvider, + shape_provider: ShapeProvider, + file_extension: str, + mimetype: str, + *plugins: NDArrayBaseIO, + ) -> None: + self.fileio = fileio + self._path_provider = path_provider + self._name_provider = name_provider + self._shape_provider = shape_provider + self._file_extension = file_extension + self._mimetype = mimetype + self._last_emitted = 0 + self._emitted_resource = False + + self._plugins = plugins + self._capture_status: Optional[AsyncStatus] = None + self._multiplier = 1 + + async def collect_frame_info(self): + detector_shape = tuple(await self._shape_provider()) + self._multiplier = multiplier + outer_shape = (multiplier,) if multiplier > 1 else () + frame_shape = detector_shape[:-1] if len(detector_shape) > 0 else [] + dtype_numpy = ( + convert_ad_dtype_to_np(detector_shape[-1]) + if len(detector_shape) > 0 + else "" + ) + return outer_shape, frame_shape, dtype_numpy + + async def begin_capture(self) -> None: + info = self._path_provider(device_name=self.fileio.name) + + # Set the directory creation depth first, since dir creation callback happens + # when directory path PV is processed. + await self.fileio.create_directory.set(info.create_dir_depth) + + await asyncio.gather( + # See https://github.com/bluesky/ophyd-async/issues/122 + self.fileio.file_path.set(str(info.directory_path)), + self.fileio.file_name.set(info.filename), + self.fileio.file_template.set("%s/%s" + self._file_extension), + self.fileio.file_write_mode.set(FileWriteMode.stream), + # Never use custom xml layout file but use the one defined + # in the source code file NDFilefileio5LayoutXML.cpp + self.fileio.xml_file_name.set(""), + ) + + assert ( + await self.fileio.file_path_exists.get_value() + ), f"File path {info.directory_path} for hdf plugin does not exist" + + # Overwrite num_capture to go forever + await self.fileio.num_capture.set(0) + # Wait for it to start, stashing the status that tells us when it finishes + self._capture_status = await set_and_wait_for_value(self.fileio.capture, True) + + async def open(self, multiplier: int = 1) -> Dict[str, DataKey]: + + self._emitted_resource = False + self._last_emitted = 0 + outer_shape, frame_shape, dtype_numpy = self.collect_frame_info() + + name = self._name_provider() + + describe = { + self._name_provider(): DataKey( + source=self._name_provider(), + shape=outer_shape + tuple(frame_shape), + dtype="array", + dtype_numpy=dtype_numpy, + external="STREAM:", + ) + } + return describe + + async def observe_indices_written( + self, timeout=DEFAULT_TIMEOUT + ) -> AsyncGenerator[int, None]: + """Wait until a specific index is ready to be collected""" + async for num_captured in observe_value(self.fileio.num_captured, timeout): + yield num_captured // self._multiplier + + async def get_indices_written(self) -> int: + num_captured = await self.fileio.num_captured.get_value() + return num_captured // self._multiplier + + async def collect_stream_docs( + self, indices_written: int + ) -> AsyncIterator[StreamAsset]: + if indices_written: + if not self._emitted_resource: + file_path = Path(await self.fileio.file_path.get_value()) + filename_template = Path(await self.fileio.file_template.get_value()) + + path_template = file_path / filename_template + # stream resource says "here is a dataset", + # stream datum says "here are N frames in that stream resource", + # you get one stream resource and many stream datums per scan + sres = { + "mimetype": self._mimetype, + "uri": path_template, + "data_key": self._name_provider(), + "uid": None, + "validate": True, + } + yield "stream_resource", sres + + if indices_written > self._last_emitted: + doc = { + "indices" : { + "start": self._last_emitted, + "stop": indices_written, + } + } + self._last_emitted = indices_written + yield "stream_datum", doc + + async def close(self): + # Already done a caput callback in _capture_status, so can't do one here + await self.fileio.capture.set(False, wait=False) + await wait_for_value(self.fileio.capture, False, DEFAULT_TIMEOUT) + if self._capture_status: + # We kicked off an open, so wait for it to return + await self._capture_status + + @property + def hints(self) -> Hints: + return {"fields": [self._name_provider()]} diff --git a/src/ophyd_async/epics/adcore/_hdf_writer.py b/src/ophyd_async/epics/adcore/_hdf_writer.py index a58eec49e1..fee84a18ff 100644 --- a/src/ophyd_async/epics/adcore/_hdf_writer.py +++ b/src/ophyd_async/epics/adcore/_hdf_writer.py @@ -20,6 +20,7 @@ ) from ._core_io import NDArrayBaseIO, NDFileHDFIO +from ._core_writer import ADWriter from ._utils import ( FileWriteMode, convert_ad_dtype_to_np, @@ -28,25 +29,16 @@ ) -class ADHDFWriter(DetectorWriter): +class ADHDFWriter(ADWriter): def __init__( self, - hdf: NDFileHDFIO, - path_provider: PathProvider, - name_provider: NameProvider, - shape_provider: ShapeProvider, - *plugins: NDArrayBaseIO, + *args, ) -> None: - self.hdf = hdf - self._path_provider = path_provider - self._name_provider = name_provider - self._shape_provider = shape_provider + super().__init__(*args) + self.hdf = self.fileio - self._plugins = plugins - self._capture_status: Optional[AsyncStatus] = None self._datasets: List[HDFDataset] = [] self._file: Optional[HDFFile] = None - self._multiplier = 1 async def open(self, multiplier: int = 1) -> Dict[str, DataKey]: self._file = None diff --git a/src/ophyd_async/epics/adcore/_tiff_writer.py b/src/ophyd_async/epics/adcore/_tiff_writer.py new file mode 100644 index 0000000000..2836f94bec --- /dev/null +++ b/src/ophyd_async/epics/adcore/_tiff_writer.py @@ -0,0 +1,63 @@ +import asyncio +from pathlib import Path +from typing import AsyncGenerator, AsyncIterator, Dict, List, Optional +from xml.etree import ElementTree as ET + +from bluesky.protocols import DataKey, Hints, StreamAsset + +from ophyd_async.core import ( + DEFAULT_TIMEOUT, + AsyncStatus, + DetectorWriter, + HDFDataset, + HDFFile, + NameProvider, + PathProvider, + ShapeProvider, + observe_value, + set_and_wait_for_value, + wait_for_value, +) + +from ._core_io import NDArrayBaseIO, NDFileHDFIO +from ._core_writer import ADWriter +from ._utils import ( + FileWriteMode, + convert_ad_dtype_to_np, + convert_param_dtype_to_np, + convert_pv_dtype_to_np, +) + + +class ADHDFWriter(ADWriter): + def __init__( + self, + *args, + ) -> None: + super().__init__(*args) + self.hdf = self.fileio + + self._datasets: List[HDFDataset] = [] + self._file: Optional[HDFFile] = None + + async def open(self, multiplier: int = 1) -> Dict[str, DataKey]: + self._file = None + info = self._path_provider(device_name=self.hdf.name) + + # Set the directory creation depth first, since dir creation callback happens + # when directory path PV is processed. + await self.hdf.create_directory.set(info.create_dir_depth) + + await asyncio.gather( + self.hdf.num_extra_dims.set(0), + self.hdf.lazy_open.set(True), + self.hdf.swmr_mode.set(True), + # See https://github.com/bluesky/ophyd-async/issues/122 + self.hdf.file_path.set(str(info.directory_path)), + self.hdf.file_name.set(info.filename), + self.hdf.file_template.set("%s/%s.h5"), + self.hdf.file_write_mode.set(FileWriteMode.stream), + # Never use custom xml layout file but use the one defined + # in the source code file NDFileHDF5LayoutXML.cpp + self.hdf.xml_file_name.set(""), + ) From f36ec3a61089d3fe0b67b99ef5ea239f9db97bab Mon Sep 17 00:00:00 2001 From: Jakub Wlodek Date: Tue, 8 Oct 2024 09:46:12 -0400 Subject: [PATCH 02/60] Continue working on tiff writer --- src/ophyd_async/epics/adcore/__init__.py | 4 + src/ophyd_async/epics/adcore/_core_io.py | 11 +- src/ophyd_async/epics/adcore/_core_writer.py | 136 +++++++++++-------- src/ophyd_async/epics/adcore/_hdf_writer.py | 13 +- src/ophyd_async/epics/adcore/_tiff_writer.py | 59 +------- 5 files changed, 94 insertions(+), 129 deletions(-) diff --git a/src/ophyd_async/epics/adcore/__init__.py b/src/ophyd_async/epics/adcore/__init__.py index f0878443d8..ebb5de441b 100644 --- a/src/ophyd_async/epics/adcore/__init__.py +++ b/src/ophyd_async/epics/adcore/__init__.py @@ -3,6 +3,7 @@ DetectorState, NDArrayBaseIO, NDFileHDFIO, + NDFileIO, NDPluginStatsIO, ) from ._core_logic import ( @@ -13,6 +14,7 @@ ) from ._hdf_writer import ADHDFWriter from ._single_trigger import SingleTriggerDetector +from ._tiff_writer import ADTIFFWriter from ._utils import ( ADBaseDataType, FileWriteMode, @@ -28,12 +30,14 @@ "ADBaseIO", "DetectorState", "NDArrayBaseIO", + "NDFileIO", "NDFileHDFIO", "NDPluginStatsIO", "DEFAULT_GOOD_STATES", "ADBaseDatasetDescriber", "set_exposure_time_and_acquire_period_if_supplied", "start_acquiring_driver_and_ensure_status", + "ADTIFFWriter", "ADHDFWriter", "SingleTriggerDetector", "ADBaseDataType", diff --git a/src/ophyd_async/epics/adcore/_core_io.py b/src/ophyd_async/epics/adcore/_core_io.py index bf5682f795..9d21d37e89 100644 --- a/src/ophyd_async/epics/adcore/_core_io.py +++ b/src/ophyd_async/epics/adcore/_core_io.py @@ -10,11 +10,6 @@ from ._utils import ADBaseDataType, FileWriteMode, ImageMode -class Callback(str, Enum): - Enable = "Enable" - Disable = "Disable" - - class NDArrayBaseIO(Device): def __init__(self, prefix: str, name: str = "") -> None: self.unique_id = epics_signal_r(int, prefix + "UniqueId_RBV") @@ -32,9 +27,7 @@ def __init__(self, prefix: str, name: str = "") -> None: class NDPluginBaseIO(NDArrayBaseIO): def __init__(self, prefix: str, name: str = "") -> None: self.nd_array_port = epics_signal_rw_rbv(str, prefix + "NDArrayPort") - self.enable_callbacks = epics_signal_rw_rbv( - Callback, prefix + "EnableCallbacks" - ) + self.enable_callbacks = epics_signal_rw_rbv(bool, prefix + "EnableCallbacks") self.nd_array_address = epics_signal_rw_rbv(int, prefix + "NDArrayAddress") self.array_size0 = epics_signal_r(int, prefix + "ArraySize0_RBV") self.array_size1 = epics_signal_r(int, prefix + "ArraySize1_RBV") @@ -118,6 +111,8 @@ def __init__(self, prefix: str, name="") -> None: self.file_path_exists = epics_signal_r(bool, prefix + "FilePathExists_RBV") self.file_template = epics_signal_rw_rbv(str, prefix + "FileTemplate") self.full_file_name = epics_signal_r(str, prefix + "FullFileName_RBV") + self.file_number = epics_signal_rw(int, prefix + "FileNumber") + self.auto_increment = epics_signal_rw(bool, prefix + "AutoIncrement") self.file_write_mode = epics_signal_rw_rbv( FileWriteMode, prefix + "FileWriteMode" ) diff --git a/src/ophyd_async/epics/adcore/_core_writer.py b/src/ophyd_async/epics/adcore/_core_writer.py index e2a23f6faf..fd5851a5cd 100644 --- a/src/ophyd_async/epics/adcore/_core_writer.py +++ b/src/ophyd_async/epics/adcore/_core_writer.py @@ -1,16 +1,27 @@ - - import asyncio +from collections.abc import AsyncGenerator, AsyncIterator from pathlib import Path -from typing import AsyncGenerator, AsyncIterator, Optional +from urllib.parse import urlunparse + +from bluesky.protocols import Hints, StreamAsset +from event_model import ( + ComposeStreamResource, + DataKey, + StreamRange, +) + from ophyd_async.core._detector import DetectorWriter -from ophyd_async.core._providers import NameProvider, PathProvider, DatasetDescriber -from ophyd_async.core._signal import observe_value, wait_for_value +from ophyd_async.core._providers import DatasetDescriber, NameProvider, PathProvider +from ophyd_async.core._signal import ( + observe_value, + set_and_wait_for_value, + wait_for_value, +) from ophyd_async.core._status import AsyncStatus from ophyd_async.core._utils import DEFAULT_TIMEOUT -from ._utils import FileWriteMode + from ._core_io import NDArrayBaseIO, NDFileIO -from event_model import DataKey +from ._utils import FileWriteMode class ADWriter(DetectorWriter): @@ -20,9 +31,9 @@ def __init__( path_provider: PathProvider, name_provider: NameProvider, dataset_describer: DatasetDescriber, - file_extension: str, - mimetype: str, *plugins: NDArrayBaseIO, + file_extension: str = ".tiff", + mimetype: str = "multipart/related;type=image/tiff", ) -> None: self.fileio = fileio self._path_provider = path_provider @@ -31,26 +42,18 @@ def __init__( self._file_extension = file_extension self._mimetype = mimetype self._last_emitted = 0 - self._emitted_resource = False + self._emitted_resource = None self._plugins = plugins - self._capture_status: Optional[AsyncStatus] = None + self._capture_status: AsyncStatus | None = None self._multiplier = 1 - - async def collect_frame_info(self): - detector_shape = tuple(await self._dataset_describer()) - self._multiplier = multiplier - outer_shape = (multiplier,) if multiplier > 1 else () - frame_shape = detector_shape[:-1] if len(detector_shape) > 0 else [] - dtype_numpy = ( - convert_ad_dtype_to_np(detector_shape[-1]) - if len(detector_shape) > 0 - else "" - ) - return outer_shape, frame_shape, dtype_numpy + self._filename_template = "%s%s_%6.6d" + self._auto_increment_file_counter = True async def begin_capture(self) -> None: - info = self._path_provider(device_name=self.fileio.name) + info = self._path_provider(device_name=self._name_provider()) + + await self.fileio.enable_callbacks.set(True) # Set the directory creation depth first, since dir creation callback happens # when directory path PV is processed. @@ -60,38 +63,38 @@ async def begin_capture(self) -> None: # See https://github.com/bluesky/ophyd-async/issues/122 self.fileio.file_path.set(str(info.directory_path)), self.fileio.file_name.set(info.filename), - self.fileio.file_template.set("%s/%s" + self._file_extension), + self.fileio.file_template.set( + self._filename_template + self._file_extension + ), self.fileio.file_write_mode.set(FileWriteMode.stream), - # Never use custom xml layout file but use the one defined - # in the source code file NDFilefileio5LayoutXML.cpp - self.fileio.xml_file_name.set(""), + self.fileio.auto_increment.set(True), ) assert ( await self.fileio.file_path_exists.get_value() - ), f"File path {info.directory_path} for hdf plugin does not exist" + ), f"File path {info.directory_path} for file plugin does not exist!" # Overwrite num_capture to go forever await self.fileio.num_capture.set(0) # Wait for it to start, stashing the status that tells us when it finishes self._capture_status = await set_and_wait_for_value(self.fileio.capture, True) - async def open(self, multiplier: int = 1) -> Dict[str, DataKey]: - - self._emitted_resource = False + async def open(self, multiplier: int = 1) -> dict[str, DataKey]: + self._emitted_resource = None self._last_emitted = 0 - outer_shape, frame_shape, dtype_numpy = self.collect_frame_info() + frame_shape = await self._dataset_describer.shape() + dtype_numpy = await self._dataset_describer.np_datatype() - name = self._name_provider() + await self.begin_capture() describe = { self._name_provider(): DataKey( source=self._name_provider(), - shape=outer_shape + tuple(frame_shape), + shape=frame_shape, dtype="array", dtype_numpy=dtype_numpy, external="STREAM:", - ) + ) # type: ignore } return describe @@ -112,30 +115,49 @@ async def collect_stream_docs( if indices_written: if not self._emitted_resource: file_path = Path(await self.fileio.file_path.get_value()) - filename_template = Path(await self.fileio.file_template.get_value()) - - path_template = file_path / filename_template - # stream resource says "here is a dataset", - # stream datum says "here are N frames in that stream resource", - # you get one stream resource and many stream datums per scan - sres = { - "mimetype": self._mimetype, - "uri": path_template, - "data_key": self._name_provider(), - "uid": None, - "validate": True, - } - yield "stream_resource", sres - + file_name = await self.fileio.file_name.get_value() + file_template = file_name + "_{:06d}" + self._file_extension + + frame_shape = await self._dataset_describer.shape() + + uri = urlunparse( + ( + "file", + "localhost", + str(file_path.absolute()) + "/", + "", + "", + None, + ) + ) + + bundler_composer = ComposeStreamResource() + + self._emitted_resource = bundler_composer( + mimetype=self._mimetype, + uri=uri, + data_key=self._name_provider(), + parameters={ + "chunk_shape": (1, *frame_shape), + "template": file_template, + }, + uid=None, + validate=True, + ) + + yield "stream_resource", self._emitted_resource.stream_resource_doc + + # Indices are relative to resource if indices_written > self._last_emitted: - doc = { - "indices" : { - "start": self._last_emitted, - "stop": indices_written, - } + indices: StreamRange = { + "start": self._last_emitted, + "stop": indices_written, } self._last_emitted = indices_written - yield "stream_datum", doc + yield ( + "stream_datum", + self._emitted_resource.compose_stream_datum(indices), + ) async def close(self): # Already done a caput callback in _capture_status, so can't do one here diff --git a/src/ophyd_async/epics/adcore/_hdf_writer.py b/src/ophyd_async/epics/adcore/_hdf_writer.py index 876b75440a..478c6103e1 100644 --- a/src/ophyd_async/epics/adcore/_hdf_writer.py +++ b/src/ophyd_async/epics/adcore/_hdf_writer.py @@ -1,7 +1,7 @@ import asyncio from collections.abc import AsyncGenerator, AsyncIterator from pathlib import Path -from typing import Optional, cast +from typing import cast from xml.etree import ElementTree as ET from bluesky.protocols import Hints, StreamAsset @@ -10,18 +10,14 @@ from ophyd_async.core import ( DEFAULT_TIMEOUT, AsyncStatus, - DatasetDescriber, - DetectorWriter, HDFDataset, HDFFile, - NameProvider, - PathProvider, observe_value, set_and_wait_for_value, wait_for_value, ) -from ._core_io import NDArrayBaseIO, NDFileHDFIO +from ._core_io import NDFileHDFIO from ._core_writer import ADWriter from ._utils import ( FileWriteMode, @@ -35,16 +31,15 @@ def __init__( self, *args, ) -> None: - super().__init__(*args) + super().__init__(*args, file_extension=".h5", mimetype="application/x-hdf5") self.hdf = cast(NDFileHDFIO, self.fileio) - self._file: Optional[HDFFile] = None + self._file: HDFFile | None = None self._capture_status: AsyncStatus | None = None self._datasets: list[HDFDataset] = [] self._file: HDFFile | None = None self._multiplier = 1 - async def open(self, multiplier: int = 1) -> dict[str, DataKey]: self._file = None info = self._path_provider(device_name=self._name_provider()) diff --git a/src/ophyd_async/epics/adcore/_tiff_writer.py b/src/ophyd_async/epics/adcore/_tiff_writer.py index 2836f94bec..dc1c0736f6 100644 --- a/src/ophyd_async/epics/adcore/_tiff_writer.py +++ b/src/ophyd_async/epics/adcore/_tiff_writer.py @@ -1,63 +1,12 @@ -import asyncio -from pathlib import Path -from typing import AsyncGenerator, AsyncIterator, Dict, List, Optional -from xml.etree import ElementTree as ET - -from bluesky.protocols import DataKey, Hints, StreamAsset - -from ophyd_async.core import ( - DEFAULT_TIMEOUT, - AsyncStatus, - DetectorWriter, - HDFDataset, - HDFFile, - NameProvider, - PathProvider, - ShapeProvider, - observe_value, - set_and_wait_for_value, - wait_for_value, -) - -from ._core_io import NDArrayBaseIO, NDFileHDFIO from ._core_writer import ADWriter -from ._utils import ( - FileWriteMode, - convert_ad_dtype_to_np, - convert_param_dtype_to_np, - convert_pv_dtype_to_np, -) -class ADHDFWriter(ADWriter): +class ADTIFFWriter(ADWriter): def __init__( self, *args, ) -> None: - super().__init__(*args) - self.hdf = self.fileio - - self._datasets: List[HDFDataset] = [] - self._file: Optional[HDFFile] = None - - async def open(self, multiplier: int = 1) -> Dict[str, DataKey]: - self._file = None - info = self._path_provider(device_name=self.hdf.name) - - # Set the directory creation depth first, since dir creation callback happens - # when directory path PV is processed. - await self.hdf.create_directory.set(info.create_dir_depth) - - await asyncio.gather( - self.hdf.num_extra_dims.set(0), - self.hdf.lazy_open.set(True), - self.hdf.swmr_mode.set(True), - # See https://github.com/bluesky/ophyd-async/issues/122 - self.hdf.file_path.set(str(info.directory_path)), - self.hdf.file_name.set(info.filename), - self.hdf.file_template.set("%s/%s.h5"), - self.hdf.file_write_mode.set(FileWriteMode.stream), - # Never use custom xml layout file but use the one defined - # in the source code file NDFileHDF5LayoutXML.cpp - self.hdf.xml_file_name.set(""), + super().__init__( + *args, file_extension=".tiff", mimetype="multipart/related;type=image/tiff" ) + self.tiff = self.fileio From 83dff621ef84637f3a878ab87c063b7e9d07ae38 Mon Sep 17 00:00:00 2001 From: Jakub Wlodek Date: Tue, 8 Oct 2024 12:33:11 -0400 Subject: [PATCH 03/60] Further work on tiff writer, existing tests now passing. --- src/ophyd_async/epics/adcore/_core_writer.py | 49 +++++++++-------- src/ophyd_async/epics/adcore/_hdf_writer.py | 57 +++++++++++--------- src/ophyd_async/epics/adcore/_tiff_writer.py | 17 ++++-- tests/epics/adcore/test_scans.py | 6 +-- 4 files changed, 75 insertions(+), 54 deletions(-) diff --git a/src/ophyd_async/epics/adcore/_core_writer.py b/src/ophyd_async/epics/adcore/_core_writer.py index fd5851a5cd..5967f7c43a 100644 --- a/src/ophyd_async/epics/adcore/_core_writer.py +++ b/src/ophyd_async/epics/adcore/_core_writer.py @@ -20,7 +20,7 @@ from ophyd_async.core._status import AsyncStatus from ophyd_async.core._utils import DEFAULT_TIMEOUT -from ._core_io import NDArrayBaseIO, NDFileIO +from ._core_io import NDFileIO from ._utils import FileWriteMode @@ -31,11 +31,10 @@ def __init__( path_provider: PathProvider, name_provider: NameProvider, dataset_describer: DatasetDescriber, - *plugins: NDArrayBaseIO, - file_extension: str = ".tiff", - mimetype: str = "multipart/related;type=image/tiff", + file_extension: str, + mimetype: str, ) -> None: - self.fileio = fileio + self._fileio = fileio self._path_provider = path_provider self._name_provider = name_provider self._dataset_describer = dataset_describer @@ -44,44 +43,46 @@ def __init__( self._last_emitted = 0 self._emitted_resource = None - self._plugins = plugins self._capture_status: AsyncStatus | None = None self._multiplier = 1 self._filename_template = "%s%s_%6.6d" - self._auto_increment_file_counter = True async def begin_capture(self) -> None: info = self._path_provider(device_name=self._name_provider()) - await self.fileio.enable_callbacks.set(True) + await self._fileio.enable_callbacks.set(True) # Set the directory creation depth first, since dir creation callback happens # when directory path PV is processed. - await self.fileio.create_directory.set(info.create_dir_depth) + await self._fileio.create_directory.set(info.create_dir_depth) await asyncio.gather( # See https://github.com/bluesky/ophyd-async/issues/122 - self.fileio.file_path.set(str(info.directory_path)), - self.fileio.file_name.set(info.filename), - self.fileio.file_template.set( + self._fileio.file_path.set(str(info.directory_path)), + self._fileio.file_name.set(info.filename), + self._fileio.file_write_mode.set(FileWriteMode.stream), + # For non-HDF file writers, use AD file templating mechanism + # for generating multi-image datasets + self._fileio.file_template.set( self._filename_template + self._file_extension ), - self.fileio.file_write_mode.set(FileWriteMode.stream), - self.fileio.auto_increment.set(True), + self._fileio.auto_increment.set(True), + self._fileio.file_number.set(0), ) assert ( - await self.fileio.file_path_exists.get_value() + await self._fileio.file_path_exists.get_value() ), f"File path {info.directory_path} for file plugin does not exist!" # Overwrite num_capture to go forever - await self.fileio.num_capture.set(0) + await self._fileio.num_capture.set(0) # Wait for it to start, stashing the status that tells us when it finishes - self._capture_status = await set_and_wait_for_value(self.fileio.capture, True) + self._capture_status = await set_and_wait_for_value(self._fileio.capture, True) async def open(self, multiplier: int = 1) -> dict[str, DataKey]: self._emitted_resource = None self._last_emitted = 0 + self._multiplier = multiplier frame_shape = await self._dataset_describer.shape() dtype_numpy = await self._dataset_describer.np_datatype() @@ -102,11 +103,11 @@ async def observe_indices_written( self, timeout=DEFAULT_TIMEOUT ) -> AsyncGenerator[int, None]: """Wait until a specific index is ready to be collected""" - async for num_captured in observe_value(self.fileio.num_captured, timeout): + async for num_captured in observe_value(self._fileio.num_captured, timeout): yield num_captured // self._multiplier async def get_indices_written(self) -> int: - num_captured = await self.fileio.num_captured.get_value() + num_captured = await self._fileio.num_captured.get_value() return num_captured // self._multiplier async def collect_stream_docs( @@ -114,8 +115,8 @@ async def collect_stream_docs( ) -> AsyncIterator[StreamAsset]: if indices_written: if not self._emitted_resource: - file_path = Path(await self.fileio.file_path.get_value()) - file_name = await self.fileio.file_name.get_value() + file_path = Path(await self._fileio.file_path.get_value()) + file_name = await self._fileio.file_name.get_value() file_template = file_name + "_{:06d}" + self._file_extension frame_shape = await self._dataset_describer.shape() @@ -138,7 +139,9 @@ async def collect_stream_docs( uri=uri, data_key=self._name_provider(), parameters={ + # Assume that we always write 1 frame per file/chunk "chunk_shape": (1, *frame_shape), + # Include file template for reconstruction in consolidator "template": file_template, }, uid=None, @@ -161,8 +164,8 @@ async def collect_stream_docs( async def close(self): # Already done a caput callback in _capture_status, so can't do one here - await self.fileio.capture.set(False, wait=False) - await wait_for_value(self.fileio.capture, False, DEFAULT_TIMEOUT) + await self._fileio.capture.set(False, wait=False) + await wait_for_value(self._fileio.capture, False, DEFAULT_TIMEOUT) if self._capture_status: # We kicked off an open, so wait for it to return await self._capture_status diff --git a/src/ophyd_async/epics/adcore/_hdf_writer.py b/src/ophyd_async/epics/adcore/_hdf_writer.py index 478c6103e1..5a92302771 100644 --- a/src/ophyd_async/epics/adcore/_hdf_writer.py +++ b/src/ophyd_async/epics/adcore/_hdf_writer.py @@ -10,17 +10,18 @@ from ophyd_async.core import ( DEFAULT_TIMEOUT, AsyncStatus, + DatasetDescriber, HDFDataset, HDFFile, + NameProvider, + PathProvider, observe_value, - set_and_wait_for_value, wait_for_value, ) -from ._core_io import NDFileHDFIO +from ._core_io import NDArrayBaseIO, NDFileHDFIO from ._core_writer import ADWriter from ._utils import ( - FileWriteMode, convert_param_dtype_to_np, convert_pv_dtype_to_np, ) @@ -29,24 +30,37 @@ class ADHDFWriter(ADWriter): def __init__( self, - *args, + hdf: NDFileHDFIO, + path_provider: PathProvider, + name_provider: NameProvider, + dataset_describer: DatasetDescriber, + *plugins: NDArrayBaseIO, ) -> None: - super().__init__(*args, file_extension=".h5", mimetype="application/x-hdf5") - self.hdf = cast(NDFileHDFIO, self.fileio) + super().__init__( + hdf, + path_provider, + name_provider, + dataset_describer, + ".h5", + "application/x-hdf5", + ) + self.hdf = cast(NDFileHDFIO, self._fileio) self._file: HDFFile | None = None self._capture_status: AsyncStatus | None = None self._datasets: list[HDFDataset] = [] self._file: HDFFile | None = None - self._multiplier = 1 + self._plugins = plugins + self._include_file_number = False + + @property + def include_file_number(self): + return self._include_file_number async def open(self, multiplier: int = 1) -> dict[str, DataKey]: self._file = None - info = self._path_provider(device_name=self._name_provider()) - # Set the directory creation depth first, since dir creation callback happens - # when directory path PV is processed. - await self.hdf.create_directory.set(info.create_dir_depth) + # Setting HDF writer specific signals # Make sure we are using chunk auto-sizing await asyncio.gather(self.hdf.chunk_size_auto.set(True)) @@ -55,24 +69,17 @@ async def open(self, multiplier: int = 1) -> dict[str, DataKey]: self.hdf.num_extra_dims.set(0), self.hdf.lazy_open.set(True), self.hdf.swmr_mode.set(True), - # See https://github.com/bluesky/ophyd-async/issues/122 - self.hdf.file_path.set(str(info.directory_path)), - self.hdf.file_name.set(info.filename), - self.hdf.file_template.set("%s/%s.h5"), - self.hdf.file_write_mode.set(FileWriteMode.stream), - # Never use custom xml layout file but use the one defined - # in the source code file NDFileHDF5LayoutXML.cpp self.hdf.xml_file_name.set(""), ) - assert ( - await self.hdf.file_path_exists.get_value() - ), f"File path {info.directory_path} for hdf plugin does not exist" + # By default, don't add file number to filename + self._filename_template = "%s%s" + if self._include_file_number: + self._filename_template += "_%6.6d" + + # Set common AD file plugin params, begin capturing + await self.begin_capture() - # Overwrite num_capture to go forever - await self.hdf.num_capture.set(0) - # Wait for it to start, stashing the status that tells us when it finishes - self._capture_status = await set_and_wait_for_value(self.hdf.capture, True) name = self._name_provider() detector_shape = await self._dataset_describer.shape() np_dtype = await self._dataset_describer.np_datatype() diff --git a/src/ophyd_async/epics/adcore/_tiff_writer.py b/src/ophyd_async/epics/adcore/_tiff_writer.py index dc1c0736f6..24d92bea9f 100644 --- a/src/ophyd_async/epics/adcore/_tiff_writer.py +++ b/src/ophyd_async/epics/adcore/_tiff_writer.py @@ -1,12 +1,23 @@ +from ophyd_async.core import DatasetDescriber, NameProvider, PathProvider + +from ._core_io import NDFileIO from ._core_writer import ADWriter class ADTIFFWriter(ADWriter): def __init__( self, - *args, + fileio: NDFileIO, + path_provider: PathProvider, + name_provider: NameProvider, + dataset_describer: DatasetDescriber, ) -> None: super().__init__( - *args, file_extension=".tiff", mimetype="multipart/related;type=image/tiff" + fileio, + path_provider, + name_provider, + dataset_describer, + ".tiff", + "multipart/related;type=image/tiff", ) - self.tiff = self.fileio + self.tiff = self._fileio diff --git a/tests/epics/adcore/test_scans.py b/tests/epics/adcore/test_scans.py index 4757b2f8b3..b6f9d10f11 100644 --- a/tests/epics/adcore/test_scans.py +++ b/tests/epics/adcore/test_scans.py @@ -67,9 +67,9 @@ def writer(RE, static_path_provider, tmp_path: Path) -> adcore.ADHDFWriter: return adcore.ADHDFWriter( hdf, - path_provider=static_path_provider, - name_provider=lambda: "test", - dataset_describer=AsyncMock(), + static_path_provider, + lambda: "test", + AsyncMock(), ) From 1a52a21eff14fd59b6d4ceadda62cb07ed55613f Mon Sep 17 00:00:00 2001 From: Jakub Wlodek Date: Tue, 8 Oct 2024 12:39:56 -0400 Subject: [PATCH 04/60] Remove functions moved to superclas from hdf writer --- src/ophyd_async/epics/adcore/_hdf_writer.py | 21 +++------------------ 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/src/ophyd_async/epics/adcore/_hdf_writer.py b/src/ophyd_async/epics/adcore/_hdf_writer.py index 5a92302771..b26768d06e 100644 --- a/src/ophyd_async/epics/adcore/_hdf_writer.py +++ b/src/ophyd_async/epics/adcore/_hdf_writer.py @@ -1,5 +1,5 @@ import asyncio -from collections.abc import AsyncGenerator, AsyncIterator +from collections.abc import AsyncIterator from pathlib import Path from typing import cast from xml.etree import ElementTree as ET @@ -9,13 +9,11 @@ from ophyd_async.core import ( DEFAULT_TIMEOUT, - AsyncStatus, DatasetDescriber, HDFDataset, HDFFile, NameProvider, PathProvider, - observe_value, wait_for_value, ) @@ -45,16 +43,14 @@ def __init__( "application/x-hdf5", ) self.hdf = cast(NDFileHDFIO, self._fileio) - - self._file: HDFFile | None = None - self._capture_status: AsyncStatus | None = None + self._plugins = plugins self._datasets: list[HDFDataset] = [] self._file: HDFFile | None = None - self._plugins = plugins self._include_file_number = False @property def include_file_number(self): + """Boolean property to toggle AD file number suffix""" return self._include_file_number async def open(self, multiplier: int = 1) -> dict[str, DataKey]: @@ -142,17 +138,6 @@ async def open(self, multiplier: int = 1) -> dict[str, DataKey]: } return describe - async def observe_indices_written( - self, timeout=DEFAULT_TIMEOUT - ) -> AsyncGenerator[int, None]: - """Wait until a specific index is ready to be collected""" - async for num_captured in observe_value(self.hdf.num_captured, timeout): - yield num_captured // self._multiplier - - async def get_indices_written(self) -> int: - num_captured = await self.hdf.num_captured.get_value() - return num_captured // self._multiplier - async def collect_stream_docs( self, indices_written: int ) -> AsyncIterator[StreamAsset]: From 489cfd8818106fe0868c7b975079e31d20fc650d Mon Sep 17 00:00:00 2001 From: Jakub Wlodek Date: Wed, 9 Oct 2024 17:36:45 -0400 Subject: [PATCH 05/60] Significant re-org and simplification of ad classes --- src/ophyd_async/epics/adcore/__init__.py | 14 +- .../epics/adcore/_core_detector.py | 70 ++++++++ src/ophyd_async/epics/adcore/_core_logic.py | 166 +++++++++++------- src/ophyd_async/epics/adcore/_core_writer.py | 48 ++--- src/ophyd_async/epics/adcore/_hdf_writer.py | 2 +- src/ophyd_async/epics/adcore/_tiff_writer.py | 2 +- src/ophyd_async/epics/adcore/_utils.py | 5 +- 7 files changed, 206 insertions(+), 101 deletions(-) create mode 100644 src/ophyd_async/epics/adcore/_core_detector.py diff --git a/src/ophyd_async/epics/adcore/__init__.py b/src/ophyd_async/epics/adcore/__init__.py index ebb5de441b..192138b441 100644 --- a/src/ophyd_async/epics/adcore/__init__.py +++ b/src/ophyd_async/epics/adcore/__init__.py @@ -9,12 +9,13 @@ from ._core_logic import ( DEFAULT_GOOD_STATES, ADBaseDatasetDescriber, - set_exposure_time_and_acquire_period_if_supplied, - start_acquiring_driver_and_ensure_status, + ADBaseController ) +from ._core_detector import AreaDetector +from ._core_writer import ADWriter from ._hdf_writer import ADHDFWriter -from ._single_trigger import SingleTriggerDetector from ._tiff_writer import ADTIFFWriter +from ._single_trigger import SingleTriggerDetector from ._utils import ( ADBaseDataType, FileWriteMode, @@ -35,10 +36,11 @@ "NDPluginStatsIO", "DEFAULT_GOOD_STATES", "ADBaseDatasetDescriber", - "set_exposure_time_and_acquire_period_if_supplied", - "start_acquiring_driver_and_ensure_status", - "ADTIFFWriter", + "ADBaseController", + "AreaDetector", + "ADWriter", "ADHDFWriter", + "ADTIFFWriter", "SingleTriggerDetector", "ADBaseDataType", "FileWriteMode", diff --git a/src/ophyd_async/epics/adcore/_core_detector.py b/src/ophyd_async/epics/adcore/_core_detector.py new file mode 100644 index 0000000000..417fa0d72b --- /dev/null +++ b/src/ophyd_async/epics/adcore/_core_detector.py @@ -0,0 +1,70 @@ +from typing import Sequence, cast +from ophyd_async.core import StandardDetector +from ophyd_async.core import PathProvider +from ophyd_async.core import SignalR + +from bluesky.protocols import HasHints, Hints + +from ._core_logic import ADBaseController, ADBaseDatasetDescriber + +from ._core_writer import ADWriter + +from ._core_writer import ADWriter +from ._core_io import NDFileHDFIO, NDFileIO, ADBaseIO +from ._hdf_writer import ADHDFWriter +from ._tiff_writer import ADTIFFWriter + + +def get_io_class_for_writer(writer_class: type[ADWriter]): + writer_to_io_map = { + ADWriter: NDFileIO, + ADHDFWriter: NDFileHDFIO, + ADTIFFWriter: NDFileIO, + } + return writer_to_io_map[writer_class] + + +class AreaDetector(StandardDetector, HasHints): + _controller: ADBaseController + _writer: ADWriter + + def __init__( + self, + prefix: str, + path_provider: PathProvider, + writer_class: type[ADWriter]=ADWriter, + writer_suffix: str="", + controller_class: type[ADBaseController]=ADBaseController, + drv_class: type[ADBaseIO]=ADBaseIO, + drv_suffix:str="cam1:", + name: str = "", + config_sigs: Sequence[SignalR] = (), + **kwargs, + ): + self.drv = drv_class(prefix + drv_suffix) + self._fileio = get_io_class_for_writer(writer_class)(prefix + writer_suffix) + + super().__init__( + controller_class(self.drv, **kwargs), + writer_class( + self._fileio, + path_provider, + lambda: self.name, + ADBaseDatasetDescriber(self.drv), + ), + config_sigs=(self.drv.acquire_period, self.drv.acquire_time, *config_sigs), + name=name, + ) + + @property + def controller(self) -> ADBaseController: + return cast(ADBaseController, self._controller) + + + @property + def writer(self) -> ADWriter: + return cast(ADWriter, self._writer) + + @property + def hints(self) -> Hints: + return self._writer.hints \ No newline at end of file diff --git a/src/ophyd_async/epics/adcore/_core_logic.py b/src/ophyd_async/epics/adcore/_core_logic.py index 3c12c293f9..278004b5e7 100644 --- a/src/ophyd_async/epics/adcore/_core_logic.py +++ b/src/ophyd_async/epics/adcore/_core_logic.py @@ -7,9 +7,13 @@ DetectorControl, set_and_wait_for_value, ) -from ophyd_async.epics.adcore._utils import convert_ad_dtype_to_np +from ophyd_async.core._detector import DetectorTrigger, TriggerInfo +from ophyd_async.core._signal import wait_for_value +from ophyd_async.core._utils import T +from ._core_io import DetectorState, ADBaseIO +from ._utils import ImageMode, convert_ad_dtype_to_np, stop_busy_record + -from ._core_io import ADBaseIO, DetectorState # Default set of states that we should consider "good" i.e. the acquisition # is complete and went well @@ -33,74 +37,100 @@ async def shape(self) -> tuple[int, int]: return shape -async def set_exposure_time_and_acquire_period_if_supplied( - controller: DetectorControl, - driver: ADBaseIO, - exposure: float | None = None, - timeout: float = DEFAULT_TIMEOUT, -) -> None: - """ - Sets the exposure time if it is not None and the acquire period to the - exposure time plus the deadtime. This is expected behavior for most - AreaDetectors, but some may require more specialized handling. - - Parameters - ---------- - controller: - Controller that can supply a deadtime. - driver: - The driver to start acquiring. Must subclass ADBaseIO. - exposure: - Desired exposure time, this is a noop if it is None. - timeout: - How long to wait for the exposure time and acquire period to be set. - """ - if exposure is not None: - full_frame_time = exposure + controller.get_deadtime(exposure) +class ADBaseController(DetectorControl): + + def __init__( + self, + driver: ADBaseIO, + good_states: frozenset[DetectorState] = DEFAULT_GOOD_STATES, + ) -> None: + self._driver = driver + self.good_states = good_states + self.frame_timeout = DEFAULT_TIMEOUT + self._arm_status: AsyncStatus | None = None + + @property + def driver(self) -> ADBaseIO: + return self._driver + + def get_deadtime(self, exposure: float | None) -> float: + return 0.002 + + async def prepare(self, trigger_info: TriggerInfo): + assert ( + trigger_info.trigger == DetectorTrigger.internal + ), "fly scanning (i.e. external triggering) is not supported for this device" + self.frame_timeout = ( + DEFAULT_TIMEOUT + await self._driver.acquire_time.get_value() + ) await asyncio.gather( - driver.acquire_time.set(exposure, timeout=timeout), - driver.acquire_period.set(full_frame_time, timeout=timeout), + self._driver.num_images.set(trigger_info.number), + self._driver.image_mode.set(ImageMode.multiple), ) - -async def start_acquiring_driver_and_ensure_status( - driver: ADBaseIO, - good_states: frozenset[DetectorState] = frozenset(DEFAULT_GOOD_STATES), - timeout: float = DEFAULT_TIMEOUT, -) -> AsyncStatus: - """ - Start acquiring driver, raising ValueError if the detector is in a bad state. - - This sets driver.acquire to True, and waits for it to be True up to a timeout. - Then, it checks that the DetectorState PV is in DEFAULT_GOOD_STATES, and otherwise - raises a ValueError. - - Parameters - ---------- - driver: - The driver to start acquiring. Must subclass ADBaseIO. - good_states: - set of states defined in DetectorState enum which are considered good states. - timeout: - How long to wait for driver.acquire to readback True (i.e. acquiring). - - Returns - ------- - AsyncStatus: - An AsyncStatus that can be awaited to set driver.acquire to True and perform - subsequent raising (if applicable) due to detector state. - """ - - status = await set_and_wait_for_value(driver.acquire, True, timeout=timeout) - - async def complete_acquisition() -> None: - """NOTE: possible race condition here between the callback from - set_and_wait_for_value and the detector state updating.""" - await status - state = await driver.detector_state.get_value() - if state not in good_states: - raise ValueError( - f"Final detector state {state} not in valid end states: {good_states}" + async def arm(self): + self._arm_status = await self.start_acquiring_driver_and_ensure_status() + + async def wait_for_idle(self): + if self._arm_status: + await self._arm_status + + async def disarm(self): + # We can't use caput callback as we already used it in arm() and we can't have + # 2 or they will deadlock + await stop_busy_record(self._driver.acquire, False, timeout=1) + + + async def set_exposure_time_and_acquire_period_if_supplied( + self, + exposure: float | None = None, + timeout: float = DEFAULT_TIMEOUT, + ) -> None: + """ + Sets the exposure time if it is not None and the acquire period to the + exposure time plus the deadtime. This is expected behavior for most + AreaDetectors, but some may require more specialized handling. + + Parameters + ---------- + exposure: + Desired exposure time, this is a noop if it is None. + timeout: + How long to wait for the exposure time and acquire period to be set. + """ + if exposure is not None: + full_frame_time = exposure + self.get_deadtime(exposure) + await asyncio.gather( + self._driver.acquire_time.set(exposure, timeout=timeout), + self._driver.acquire_period.set(full_frame_time, timeout=timeout), ) - return AsyncStatus(complete_acquisition()) + + async def start_acquiring_driver_and_ensure_status(self) -> AsyncStatus: + """ + Start acquiring driver, raising ValueError if the detector is in a bad state. + + This sets driver.acquire to True, and waits for it to be True up to a timeout. + Then, it checks that the DetectorState PV is in DEFAULT_GOOD_STATES, and otherwise + raises a ValueError. + + Returns + ------- + AsyncStatus: + An AsyncStatus that can be awaited to set driver.acquire to True and perform + subsequent raising (if applicable) due to detector state. + """ + + status = await set_and_wait_for_value(self._driver.acquire, True, timeout=self.frame_timeout) + + async def complete_acquisition() -> None: + """NOTE: possible race condition here between the callback from + set_and_wait_for_value and the detector state updating.""" + await status + state = await self._driver.detector_state.get_value() + if state not in self.good_states: + raise ValueError( + f"Final detector state {state} not in valid end states: {self.good_states}" + ) + + return AsyncStatus(complete_acquisition()) diff --git a/src/ophyd_async/epics/adcore/_core_writer.py b/src/ophyd_async/epics/adcore/_core_writer.py index 5967f7c43a..2ed6fe1769 100644 --- a/src/ophyd_async/epics/adcore/_core_writer.py +++ b/src/ophyd_async/epics/adcore/_core_writer.py @@ -20,9 +20,8 @@ from ophyd_async.core._status import AsyncStatus from ophyd_async.core._utils import DEFAULT_TIMEOUT -from ._core_io import NDFileIO from ._utils import FileWriteMode - +from ._core_io import NDFileIO class ADWriter(DetectorWriter): def __init__( @@ -31,10 +30,10 @@ def __init__( path_provider: PathProvider, name_provider: NameProvider, dataset_describer: DatasetDescriber, - file_extension: str, - mimetype: str, + file_extension: str = "", + mimetype: str = "", ) -> None: - self._fileio = fileio + self.fileio = fileio self._path_provider = path_provider self._name_provider = name_provider self._dataset_describer = dataset_describer @@ -50,34 +49,34 @@ def __init__( async def begin_capture(self) -> None: info = self._path_provider(device_name=self._name_provider()) - await self._fileio.enable_callbacks.set(True) + await self.fileio.enable_callbacks.set(True) # Set the directory creation depth first, since dir creation callback happens # when directory path PV is processed. - await self._fileio.create_directory.set(info.create_dir_depth) + await self.fileio.create_directory.set(info.create_dir_depth) await asyncio.gather( # See https://github.com/bluesky/ophyd-async/issues/122 - self._fileio.file_path.set(str(info.directory_path)), - self._fileio.file_name.set(info.filename), - self._fileio.file_write_mode.set(FileWriteMode.stream), + self.fileio.file_path.set(str(info.directory_path)), + self.fileio.file_name.set(info.filename), + self.fileio.file_write_mode.set(FileWriteMode.stream), # For non-HDF file writers, use AD file templating mechanism # for generating multi-image datasets - self._fileio.file_template.set( + self.fileio.file_template.set( self._filename_template + self._file_extension ), - self._fileio.auto_increment.set(True), - self._fileio.file_number.set(0), + self.fileio.auto_increment.set(True), + self.fileio.file_number.set(0), ) assert ( - await self._fileio.file_path_exists.get_value() + await self.fileio.file_path_exists.get_value() ), f"File path {info.directory_path} for file plugin does not exist!" # Overwrite num_capture to go forever - await self._fileio.num_capture.set(0) + await self.fileio.num_capture.set(0) # Wait for it to start, stashing the status that tells us when it finishes - self._capture_status = await set_and_wait_for_value(self._fileio.capture, True) + self._capture_status = await set_and_wait_for_value(self.fileio.capture, True) async def open(self, multiplier: int = 1) -> dict[str, DataKey]: self._emitted_resource = None @@ -91,7 +90,7 @@ async def open(self, multiplier: int = 1) -> dict[str, DataKey]: describe = { self._name_provider(): DataKey( source=self._name_provider(), - shape=frame_shape, + shape=tuple(frame_shape), dtype="array", dtype_numpy=dtype_numpy, external="STREAM:", @@ -103,11 +102,11 @@ async def observe_indices_written( self, timeout=DEFAULT_TIMEOUT ) -> AsyncGenerator[int, None]: """Wait until a specific index is ready to be collected""" - async for num_captured in observe_value(self._fileio.num_captured, timeout): + async for num_captured in observe_value(self.fileio.num_captured, timeout): yield num_captured // self._multiplier async def get_indices_written(self) -> int: - num_captured = await self._fileio.num_captured.get_value() + num_captured = await self.fileio.num_captured.get_value() return num_captured // self._multiplier async def collect_stream_docs( @@ -115,8 +114,8 @@ async def collect_stream_docs( ) -> AsyncIterator[StreamAsset]: if indices_written: if not self._emitted_resource: - file_path = Path(await self._fileio.file_path.get_value()) - file_name = await self._fileio.file_name.get_value() + file_path = Path(await self.fileio.file_path.get_value()) + file_name = await self.fileio.file_name.get_value() file_template = file_name + "_{:06d}" + self._file_extension frame_shape = await self._dataset_describer.shape() @@ -164,8 +163,8 @@ async def collect_stream_docs( async def close(self): # Already done a caput callback in _capture_status, so can't do one here - await self._fileio.capture.set(False, wait=False) - await wait_for_value(self._fileio.capture, False, DEFAULT_TIMEOUT) + await self.fileio.capture.set(False, wait=False) + await wait_for_value(self.fileio.capture, False, DEFAULT_TIMEOUT) if self._capture_status: # We kicked off an open, so wait for it to return await self._capture_status @@ -173,3 +172,6 @@ async def close(self): @property def hints(self) -> Hints: return {"fields": [self._name_provider()]} + + + diff --git a/src/ophyd_async/epics/adcore/_hdf_writer.py b/src/ophyd_async/epics/adcore/_hdf_writer.py index b26768d06e..2481db44e1 100644 --- a/src/ophyd_async/epics/adcore/_hdf_writer.py +++ b/src/ophyd_async/epics/adcore/_hdf_writer.py @@ -42,7 +42,7 @@ def __init__( ".h5", "application/x-hdf5", ) - self.hdf = cast(NDFileHDFIO, self._fileio) + self.hdf = cast(NDFileHDFIO, self.fileio) self._plugins = plugins self._datasets: list[HDFDataset] = [] self._file: HDFFile | None = None diff --git a/src/ophyd_async/epics/adcore/_tiff_writer.py b/src/ophyd_async/epics/adcore/_tiff_writer.py index 24d92bea9f..5dd6cb9225 100644 --- a/src/ophyd_async/epics/adcore/_tiff_writer.py +++ b/src/ophyd_async/epics/adcore/_tiff_writer.py @@ -20,4 +20,4 @@ def __init__( ".tiff", "multipart/related;type=image/tiff", ) - self.tiff = self._fileio + self.tiff = self.fileio diff --git a/src/ophyd_async/epics/adcore/_utils.py b/src/ophyd_async/epics/adcore/_utils.py index a1a21b6071..d80f23aeb6 100644 --- a/src/ophyd_async/epics/adcore/_utils.py +++ b/src/ophyd_async/epics/adcore/_utils.py @@ -1,8 +1,9 @@ from dataclasses import dataclass from enum import Enum -from ophyd_async.core import DEFAULT_TIMEOUT, SignalRW, T, wait_for_value -from ophyd_async.core._signal import SignalR +from ophyd_async.core import DEFAULT_TIMEOUT, SignalRW, T, wait_for_value, SignalR + + class ADBaseDataType(str, Enum): From 83c68846c07a1e911069940e6db0337c09556f1f Mon Sep 17 00:00:00 2001 From: Jakub Wlodek Date: Wed, 9 Oct 2024 17:38:25 -0400 Subject: [PATCH 06/60] Ruff formatting --- src/ophyd_async/epics/adcore/__init__.py | 10 +++---- .../epics/adcore/_core_detector.py | 26 ++++++++----------- src/ophyd_async/epics/adcore/_core_logic.py | 21 +++++++-------- src/ophyd_async/epics/adcore/_core_writer.py | 6 ++--- src/ophyd_async/epics/adcore/_utils.py | 4 +-- 5 files changed, 26 insertions(+), 41 deletions(-) diff --git a/src/ophyd_async/epics/adcore/__init__.py b/src/ophyd_async/epics/adcore/__init__.py index 192138b441..6dd0f3226d 100644 --- a/src/ophyd_async/epics/adcore/__init__.py +++ b/src/ophyd_async/epics/adcore/__init__.py @@ -1,3 +1,4 @@ +from ._core_detector import AreaDetector from ._core_io import ( ADBaseIO, DetectorState, @@ -6,16 +7,11 @@ NDFileIO, NDPluginStatsIO, ) -from ._core_logic import ( - DEFAULT_GOOD_STATES, - ADBaseDatasetDescriber, - ADBaseController -) -from ._core_detector import AreaDetector +from ._core_logic import DEFAULT_GOOD_STATES, ADBaseController, ADBaseDatasetDescriber from ._core_writer import ADWriter from ._hdf_writer import ADHDFWriter -from ._tiff_writer import ADTIFFWriter from ._single_trigger import SingleTriggerDetector +from ._tiff_writer import ADTIFFWriter from ._utils import ( ADBaseDataType, FileWriteMode, diff --git a/src/ophyd_async/epics/adcore/_core_detector.py b/src/ophyd_async/epics/adcore/_core_detector.py index 417fa0d72b..23e3c0b126 100644 --- a/src/ophyd_async/epics/adcore/_core_detector.py +++ b/src/ophyd_async/epics/adcore/_core_detector.py @@ -1,16 +1,13 @@ -from typing import Sequence, cast -from ophyd_async.core import StandardDetector -from ophyd_async.core import PathProvider -from ophyd_async.core import SignalR +from collections.abc import Sequence +from typing import cast from bluesky.protocols import HasHints, Hints -from ._core_logic import ADBaseController, ADBaseDatasetDescriber - -from ._core_writer import ADWriter +from ophyd_async.core import PathProvider, SignalR, StandardDetector +from ._core_io import ADBaseIO, NDFileHDFIO, NDFileIO +from ._core_logic import ADBaseController, ADBaseDatasetDescriber from ._core_writer import ADWriter -from ._core_io import NDFileHDFIO, NDFileIO, ADBaseIO from ._hdf_writer import ADHDFWriter from ._tiff_writer import ADTIFFWriter @@ -32,11 +29,11 @@ def __init__( self, prefix: str, path_provider: PathProvider, - writer_class: type[ADWriter]=ADWriter, - writer_suffix: str="", - controller_class: type[ADBaseController]=ADBaseController, - drv_class: type[ADBaseIO]=ADBaseIO, - drv_suffix:str="cam1:", + writer_class: type[ADWriter] = ADWriter, + writer_suffix: str = "", + controller_class: type[ADBaseController] = ADBaseController, + drv_class: type[ADBaseIO] = ADBaseIO, + drv_suffix: str = "cam1:", name: str = "", config_sigs: Sequence[SignalR] = (), **kwargs, @@ -60,11 +57,10 @@ def __init__( def controller(self) -> ADBaseController: return cast(ADBaseController, self._controller) - @property def writer(self) -> ADWriter: return cast(ADWriter, self._writer) @property def hints(self) -> Hints: - return self._writer.hints \ No newline at end of file + return self._writer.hints diff --git a/src/ophyd_async/epics/adcore/_core_logic.py b/src/ophyd_async/epics/adcore/_core_logic.py index 278004b5e7..e3a075eca7 100644 --- a/src/ophyd_async/epics/adcore/_core_logic.py +++ b/src/ophyd_async/epics/adcore/_core_logic.py @@ -8,12 +8,9 @@ set_and_wait_for_value, ) from ophyd_async.core._detector import DetectorTrigger, TriggerInfo -from ophyd_async.core._signal import wait_for_value -from ophyd_async.core._utils import T -from ._core_io import DetectorState, ADBaseIO -from ._utils import ImageMode, convert_ad_dtype_to_np, stop_busy_record - +from ._core_io import ADBaseIO, DetectorState +from ._utils import ImageMode, convert_ad_dtype_to_np, stop_busy_record # Default set of states that we should consider "good" i.e. the acquisition # is complete and went well @@ -38,7 +35,6 @@ async def shape(self) -> tuple[int, int]: class ADBaseController(DetectorControl): - def __init__( self, driver: ADBaseIO, @@ -80,7 +76,6 @@ async def disarm(self): # 2 or they will deadlock await stop_busy_record(self._driver.acquire, False, timeout=1) - async def set_exposure_time_and_acquire_period_if_supplied( self, exposure: float | None = None, @@ -105,14 +100,13 @@ async def set_exposure_time_and_acquire_period_if_supplied( self._driver.acquire_period.set(full_frame_time, timeout=timeout), ) - async def start_acquiring_driver_and_ensure_status(self) -> AsyncStatus: """ Start acquiring driver, raising ValueError if the detector is in a bad state. This sets driver.acquire to True, and waits for it to be True up to a timeout. - Then, it checks that the DetectorState PV is in DEFAULT_GOOD_STATES, and otherwise - raises a ValueError. + Then, it checks that the DetectorState PV is in DEFAULT_GOOD_STATES, + and otherwise raises a ValueError. Returns ------- @@ -121,7 +115,9 @@ async def start_acquiring_driver_and_ensure_status(self) -> AsyncStatus: subsequent raising (if applicable) due to detector state. """ - status = await set_and_wait_for_value(self._driver.acquire, True, timeout=self.frame_timeout) + status = await set_and_wait_for_value( + self._driver.acquire, True, timeout=self.frame_timeout + ) async def complete_acquisition() -> None: """NOTE: possible race condition here between the callback from @@ -130,7 +126,8 @@ async def complete_acquisition() -> None: state = await self._driver.detector_state.get_value() if state not in self.good_states: raise ValueError( - f"Final detector state {state} not in valid end states: {self.good_states}" + f"Final detector state {state} not" + "in valid end states: {self.good_states}" ) return AsyncStatus(complete_acquisition()) diff --git a/src/ophyd_async/epics/adcore/_core_writer.py b/src/ophyd_async/epics/adcore/_core_writer.py index 2ed6fe1769..08483156e0 100644 --- a/src/ophyd_async/epics/adcore/_core_writer.py +++ b/src/ophyd_async/epics/adcore/_core_writer.py @@ -20,8 +20,9 @@ from ophyd_async.core._status import AsyncStatus from ophyd_async.core._utils import DEFAULT_TIMEOUT -from ._utils import FileWriteMode from ._core_io import NDFileIO +from ._utils import FileWriteMode + class ADWriter(DetectorWriter): def __init__( @@ -172,6 +173,3 @@ async def close(self): @property def hints(self) -> Hints: return {"fields": [self._name_provider()]} - - - diff --git a/src/ophyd_async/epics/adcore/_utils.py b/src/ophyd_async/epics/adcore/_utils.py index d80f23aeb6..6b974c047a 100644 --- a/src/ophyd_async/epics/adcore/_utils.py +++ b/src/ophyd_async/epics/adcore/_utils.py @@ -1,9 +1,7 @@ from dataclasses import dataclass from enum import Enum -from ophyd_async.core import DEFAULT_TIMEOUT, SignalRW, T, wait_for_value, SignalR - - +from ophyd_async.core import DEFAULT_TIMEOUT, SignalR, SignalRW, T, wait_for_value class ADBaseDataType(str, Enum): From 3b4f45ac00fef1cd21e82a42978f9a16b29fa7fb Mon Sep 17 00:00:00 2001 From: Jakub Wlodek Date: Wed, 9 Oct 2024 17:38:52 -0400 Subject: [PATCH 07/60] Modify ad sim classes to reflect new superclasses --- .../epics/adsimdetector/__init__.py | 5 +- src/ophyd_async/epics/adsimdetector/_sim.py | 55 +++++++++++++------ .../epics/adsimdetector/_sim_controller.py | 51 ----------------- 3 files changed, 40 insertions(+), 71 deletions(-) delete mode 100644 src/ophyd_async/epics/adsimdetector/_sim_controller.py diff --git a/src/ophyd_async/epics/adsimdetector/__init__.py b/src/ophyd_async/epics/adsimdetector/__init__.py index 9904acb2e6..ae242ffddf 100644 --- a/src/ophyd_async/epics/adsimdetector/__init__.py +++ b/src/ophyd_async/epics/adsimdetector/__init__.py @@ -1,7 +1,6 @@ -from ._sim import SimDetector -from ._sim_controller import SimController +from ._sim import SimDetector, SimDetectorTIFF __all__ = [ "SimDetector", - "SimController", + "SimDetectorTIFF", ] diff --git a/src/ophyd_async/epics/adsimdetector/_sim.py b/src/ophyd_async/epics/adsimdetector/_sim.py index acc640899a..56351a8c56 100644 --- a/src/ophyd_async/epics/adsimdetector/_sim.py +++ b/src/ophyd_async/epics/adsimdetector/_sim.py @@ -1,35 +1,56 @@ from collections.abc import Sequence -from ophyd_async.core import PathProvider, SignalR, StandardDetector +from ophyd_async.core import PathProvider, SignalR from ophyd_async.epics import adcore -from ._sim_controller import SimController +class SimDetector(adcore.AreaDetector): -class SimDetector(StandardDetector): - _controller: SimController - _writer: adcore.ADHDFWriter + def __init__( + self, + prefix: str, + path_provider: PathProvider, + hdf_suffix:str="HDF1:", + drv_suffix:str="cam1:", + name: str = "", + config_sigs: Sequence[SignalR] = (), + ): + + super().__init__( + prefix, + path_provider, + adcore.ADHDFWriter, + hdf_suffix, + adcore.ADBaseController, + adcore.ADBaseIO, + drv_suffix=drv_suffix, + name=name, + config_sigs=config_sigs, + ) + self.hdf = self._fileio + + +class SimDetectorTIFF(adcore.AreaDetector): def __init__( self, prefix: str, path_provider: PathProvider, - drv_suffix="cam1:", - hdf_suffix="HDF1:", + tiff_suffix:str="TIFF1:", + drv_suffix:str="cam1:", name: str = "", config_sigs: Sequence[SignalR] = (), ): - self.drv = adcore.ADBaseIO(prefix + drv_suffix) - self.hdf = adcore.NDFileHDFIO(prefix + hdf_suffix) super().__init__( - SimController(self.drv), - adcore.ADHDFWriter( - self.hdf, - path_provider, - lambda: self.name, - adcore.ADBaseDatasetDescriber(self.drv), - ), - config_sigs=(self.drv.acquire_period, self.drv.acquire_time, *config_sigs), + prefix, + path_provider, + adcore.ADTIFFWriter, + tiff_suffix, + adcore.ADBaseController, + adcore.ADBaseIO, + drv_suffix=drv_suffix, name=name, + config_sigs=config_sigs, ) + self.tiff = self._fileio diff --git a/src/ophyd_async/epics/adsimdetector/_sim_controller.py b/src/ophyd_async/epics/adsimdetector/_sim_controller.py deleted file mode 100644 index 10b8516ece..0000000000 --- a/src/ophyd_async/epics/adsimdetector/_sim_controller.py +++ /dev/null @@ -1,51 +0,0 @@ -import asyncio - -from ophyd_async.core import ( - DEFAULT_TIMEOUT, - DetectorControl, - DetectorTrigger, -) -from ophyd_async.core._detector import TriggerInfo -from ophyd_async.core._status import AsyncStatus -from ophyd_async.epics import adcore - - -class SimController(DetectorControl): - def __init__( - self, - driver: adcore.ADBaseIO, - good_states: frozenset[adcore.DetectorState] = adcore.DEFAULT_GOOD_STATES, - ) -> None: - self.driver = driver - self.good_states = good_states - self.frame_timeout: float - self._arm_status: AsyncStatus | None = None - - def get_deadtime(self, exposure: float | None) -> float: - return 0.002 - - async def prepare(self, trigger_info: TriggerInfo): - assert ( - trigger_info.trigger == DetectorTrigger.internal - ), "fly scanning (i.e. external triggering) is not supported for this device" - self.frame_timeout = ( - DEFAULT_TIMEOUT + await self.driver.acquire_time.get_value() - ) - await asyncio.gather( - self.driver.num_images.set(trigger_info.number), - self.driver.image_mode.set(adcore.ImageMode.multiple), - ) - - async def arm(self): - self._arm_status = await adcore.start_acquiring_driver_and_ensure_status( - self.driver, good_states=self.good_states, timeout=self.frame_timeout - ) - - async def wait_for_idle(self): - if self._arm_status: - await self._arm_status - - async def disarm(self): - # We can't use caput callback as we already used it in arm() and we can't have - # 2 or they will deadlock - await adcore.stop_busy_record(self.driver.acquire, False, timeout=1) From 7175b30e79aa8b5985df140c56aebb465908911a Mon Sep 17 00:00:00 2001 From: Jakub Wlodek Date: Wed, 9 Oct 2024 17:40:09 -0400 Subject: [PATCH 08/60] Modify vimba and kinetix classes --- src/ophyd_async/epics/adkinetix/__init__.py | 3 +- src/ophyd_async/epics/adkinetix/_kinetix.py | 61 +++++++++++++------ .../epics/adkinetix/_kinetix_controller.py | 37 +++++------ src/ophyd_async/epics/advimba/_vimba.py | 60 ++++++++++++------ .../epics/advimba/_vimba_controller.py | 41 +++++-------- 5 files changed, 114 insertions(+), 88 deletions(-) diff --git a/src/ophyd_async/epics/adkinetix/__init__.py b/src/ophyd_async/epics/adkinetix/__init__.py index 7747be3356..e69dc95136 100644 --- a/src/ophyd_async/epics/adkinetix/__init__.py +++ b/src/ophyd_async/epics/adkinetix/__init__.py @@ -1,9 +1,10 @@ -from ._kinetix import KinetixDetector +from ._kinetix import KinetixDetector, KinetixDetectorTIFF from ._kinetix_controller import KinetixController from ._kinetix_io import KinetixDriverIO __all__ = [ "KinetixDetector", + "KinetixDetectorTIFF", "KinetixController", "KinetixDriverIO", ] diff --git a/src/ophyd_async/epics/adkinetix/_kinetix.py b/src/ophyd_async/epics/adkinetix/_kinetix.py index 1a1f8ae987..b53f43582b 100644 --- a/src/ophyd_async/epics/adkinetix/_kinetix.py +++ b/src/ophyd_async/epics/adkinetix/_kinetix.py @@ -1,21 +1,18 @@ -from bluesky.protocols import HasHints, Hints +from collections.abc import Sequence -from ophyd_async.core import PathProvider, StandardDetector +from ophyd_async.core import PathProvider, SignalR from ophyd_async.epics import adcore from ._kinetix_controller import KinetixController from ._kinetix_io import KinetixDriverIO -class KinetixDetector(StandardDetector, HasHints): +class KinetixDetector(adcore.AreaDetector): """ Ophyd-async implementation of an ADKinetix Detector. https://github.com/NSLS-II/ADKinetix """ - _controller: KinetixController - _writer: adcore.ADHDFWriter - def __init__( self, prefix: str, @@ -23,22 +20,46 @@ def __init__( drv_suffix="cam1:", hdf_suffix="HDF1:", name="", + config_sigs: Sequence[SignalR] = (), ): - self.drv = KinetixDriverIO(prefix + drv_suffix) - self.hdf = adcore.NDFileHDFIO(prefix + hdf_suffix) - super().__init__( - KinetixController(self.drv), - adcore.ADHDFWriter( - self.hdf, - path_provider, - lambda: self.name, - adcore.ADBaseDatasetDescriber(self.drv), - ), - config_sigs=(self.drv.acquire_time,), + prefix, + path_provider, + adcore.ADHDFWriter, + hdf_suffix, + KinetixController, + KinetixDriverIO, + drv_suffix=drv_suffix, name=name, + config_sigs=config_sigs, ) + self.hdf = self._fileio + - @property - def hints(self) -> Hints: - return self._writer.hints +class KinetixDetectorTIFF(adcore.AreaDetector): + """ + Ophyd-async implementation of an ADKinetix Detector. + https://github.com/NSLS-II/ADKinetix + """ + + def __init__( + self, + prefix: str, + path_provider: PathProvider, + drv_suffix="cam1:", + tiff_suffix="TIFF1:", + name="", + config_sigs: Sequence[SignalR] = (), + ): + super().__init__( + prefix, + path_provider, + adcore.ADTIFFWriter, + tiff_suffix, + KinetixController, + KinetixDriverIO, + drv_suffix=drv_suffix, + name=name, + config_sigs=config_sigs, + ) + self.tiff = self._fileio diff --git a/src/ophyd_async/epics/adkinetix/_kinetix_controller.py b/src/ophyd_async/epics/adkinetix/_kinetix_controller.py index acf95850ef..bcf26e85a6 100644 --- a/src/ophyd_async/epics/adkinetix/_kinetix_controller.py +++ b/src/ophyd_async/epics/adkinetix/_kinetix_controller.py @@ -1,8 +1,8 @@ import asyncio +from typing import cast -from ophyd_async.core import DetectorControl, DetectorTrigger +from ophyd_async.core import DetectorTrigger from ophyd_async.core._detector import TriggerInfo -from ophyd_async.core._status import AsyncStatus from ophyd_async.epics import adcore from ._kinetix_io import KinetixDriverIO, KinetixTriggerMode @@ -15,37 +15,30 @@ } -class KinetixController(DetectorControl): +class KinetixController(adcore.ADBaseController): def __init__( self, driver: KinetixDriverIO, ) -> None: - self._drv = driver - self._arm_status: AsyncStatus | None = None + super().__init__(driver) + + @property + def driver(self) -> KinetixDriverIO: + return cast(KinetixDriverIO, self._driver) def get_deadtime(self, exposure: float | None) -> float: - return 0.001 + return 0.0000001 async def prepare(self, trigger_info: TriggerInfo): await asyncio.gather( - self._drv.trigger_mode.set(KINETIX_TRIGGER_MODE_MAP[trigger_info.trigger]), - self._drv.num_images.set(trigger_info.number), - self._drv.image_mode.set(adcore.ImageMode.multiple), + self.driver.trigger_mode.set( + KINETIX_TRIGGER_MODE_MAP[trigger_info.trigger] + ), + self.driver.num_images.set(trigger_info.number), + self.driver.image_mode.set(adcore.ImageMode.multiple), ) if trigger_info.livetime is not None and trigger_info.trigger not in [ DetectorTrigger.variable_gate, DetectorTrigger.constant_gate, ]: - await self._drv.acquire_time.set(trigger_info.livetime) - - async def arm(self): - self._arm_status = await adcore.start_acquiring_driver_and_ensure_status( - self._drv - ) - - async def wait_for_idle(self): - if self._arm_status: - await self._arm_status - - async def disarm(self): - await adcore.stop_busy_record(self._drv.acquire, False, timeout=1) + await self.driver.acquire_time.set(trigger_info.livetime) diff --git a/src/ophyd_async/epics/advimba/_vimba.py b/src/ophyd_async/epics/advimba/_vimba.py index 1dadfc3242..4f48bcaefd 100644 --- a/src/ophyd_async/epics/advimba/_vimba.py +++ b/src/ophyd_async/epics/advimba/_vimba.py @@ -1,20 +1,17 @@ -from bluesky.protocols import HasHints, Hints +from collections.abc import Sequence -from ophyd_async.core import PathProvider, StandardDetector +from ophyd_async.core import PathProvider, SignalR from ophyd_async.epics import adcore from ._vimba_controller import VimbaController from ._vimba_io import VimbaDriverIO -class VimbaDetector(StandardDetector, HasHints): +class VimbaDetector(adcore.AreaDetector): """ Ophyd-async implementation of an ADVimba Detector. """ - _controller: VimbaController - _writer: adcore.ADHDFWriter - def __init__( self, prefix: str, @@ -22,22 +19,45 @@ def __init__( drv_suffix="cam1:", hdf_suffix="HDF1:", name="", + config_sigs: Sequence[SignalR] = (), ): - self.drv = VimbaDriverIO(prefix + drv_suffix) - self.hdf = adcore.NDFileHDFIO(prefix + hdf_suffix) - super().__init__( - VimbaController(self.drv), - adcore.ADHDFWriter( - self.hdf, - path_provider, - lambda: self.name, - adcore.ADBaseDatasetDescriber(self.drv), - ), - config_sigs=(self.drv.acquire_time,), + prefix, + path_provider, + adcore.ADHDFWriter, + hdf_suffix, + VimbaController, + VimbaDriverIO, + drv_suffix=drv_suffix, name=name, + config_sigs=config_sigs, ) + self.hdf = self._fileio + - @property - def hints(self) -> Hints: - return self._writer.hints +class VimbaDetectorTIFF(adcore.AreaDetector): + """ + Ophyd-async implementation of an ADVimba Detector. + """ + + def __init__( + self, + prefix: str, + path_provider: PathProvider, + drv_suffix="cam1:", + tiff_suffix="TIFF1:", + name="", + config_sigs: Sequence[SignalR] = (), + ): + super().__init__( + prefix, + path_provider, + adcore.ADTIFFWriter, + tiff_suffix, + VimbaController, + VimbaDriverIO, + drv_suffix=drv_suffix, + name=name, + config_sigs=config_sigs, + ) + self.tiff = self._fileio diff --git a/src/ophyd_async/epics/advimba/_vimba_controller.py b/src/ophyd_async/epics/advimba/_vimba_controller.py index f9ce2a8d02..afd103bdbc 100644 --- a/src/ophyd_async/epics/advimba/_vimba_controller.py +++ b/src/ophyd_async/epics/advimba/_vimba_controller.py @@ -1,8 +1,8 @@ import asyncio +from typing import cast -from ophyd_async.core import DetectorControl, DetectorTrigger +from ophyd_async.core import DetectorTrigger from ophyd_async.core._detector import TriggerInfo -from ophyd_async.core._status import AsyncStatus from ophyd_async.epics import adcore from ._vimba_io import VimbaDriverIO, VimbaExposeOutMode, VimbaOnOff, VimbaTriggerSource @@ -22,42 +22,33 @@ } -class VimbaController(DetectorControl): +class VimbaController(adcore.ADBaseController): def __init__( self, driver: VimbaDriverIO, ) -> None: - self._drv = driver - self._arm_status: AsyncStatus | None = None + super().__init__(driver) + + @property + def driver(self) -> VimbaDriverIO: + return cast(VimbaDriverIO, self._driver) def get_deadtime(self, exposure: float | None) -> float: - return 0.001 + return 0.00001 async def prepare(self, trigger_info: TriggerInfo): await asyncio.gather( - self._drv.trigger_mode.set(TRIGGER_MODE[trigger_info.trigger]), - self._drv.exposure_mode.set(EXPOSE_OUT_MODE[trigger_info.trigger]), - self._drv.num_images.set(trigger_info.number), - self._drv.image_mode.set(adcore.ImageMode.multiple), + self.driver.trigger_mode.set(TRIGGER_MODE[trigger_info.trigger]), + self.driver.exposure_mode.set(EXPOSE_OUT_MODE[trigger_info.trigger]), + self.driver.num_images.set(trigger_info.number), + self.driver.image_mode.set(adcore.ImageMode.multiple), ) if trigger_info.livetime is not None and trigger_info.trigger not in [ DetectorTrigger.variable_gate, DetectorTrigger.constant_gate, ]: - await self._drv.acquire_time.set(trigger_info.livetime) + await self.driver.acquire_time.set(trigger_info.livetime) if trigger_info.trigger != DetectorTrigger.internal: - self._drv.trigger_source.set(VimbaTriggerSource.line1) + self.driver.trigger_source.set(VimbaTriggerSource.line1) else: - self._drv.trigger_source.set(VimbaTriggerSource.freerun) - - async def arm(self): - self._arm_status = await adcore.start_acquiring_driver_and_ensure_status( - self._drv - ) - - async def wait_for_idle(self): - if self._arm_status: - await self._arm_status - - async def disarm(self): - await adcore.stop_busy_record(self._drv.acquire, False, timeout=1) + self.driver.trigger_source.set(VimbaTriggerSource.freerun) From faf53d6f6d87e67fa9f0f900fb103cbd4a11c2bc Mon Sep 17 00:00:00 2001 From: Jakub Wlodek Date: Wed, 9 Oct 2024 17:40:55 -0400 Subject: [PATCH 09/60] Modify aravis and pilatus classes --- src/ophyd_async/epics/adaravis/_aravis.py | 77 +++++++++++++------ .../epics/adaravis/_aravis_controller.py | 34 +++----- src/ophyd_async/epics/adpilatus/_pilatus.py | 65 ++++++++++------ .../epics/adpilatus/_pilatus_controller.py | 36 ++++----- 4 files changed, 123 insertions(+), 89 deletions(-) diff --git a/src/ophyd_async/epics/adaravis/_aravis.py b/src/ophyd_async/epics/adaravis/_aravis.py index 3a5b9634d7..d72a2f3505 100644 --- a/src/ophyd_async/epics/adaravis/_aravis.py +++ b/src/ophyd_async/epics/adaravis/_aravis.py @@ -1,24 +1,21 @@ -from typing import get_args +from collections.abc import Sequence +from typing import cast, get_args -from bluesky.protocols import HasHints, Hints - -from ophyd_async.core import PathProvider, StandardDetector +from ophyd_async.core import PathProvider +from ophyd_async.core._signal import SignalR from ophyd_async.epics import adcore from ._aravis_controller import AravisController from ._aravis_io import AravisDriverIO -class AravisDetector(StandardDetector, HasHints): +class AravisDetector(adcore.AreaDetector): """ Ophyd-async implementation of an ADAravis Detector. The detector may be configured for an external trigger on a GPIO port, which must be done prior to preparing the detector """ - _controller: AravisController - _writer: adcore.ADHDFWriter - def __init__( self, prefix: str, @@ -27,24 +24,28 @@ def __init__( hdf_suffix="HDF1:", name="", gpio_number: AravisController.GPIO_NUMBER = 1, + config_sigs: Sequence[SignalR] = (), ): - self.drv = AravisDriverIO(prefix + drv_suffix) - self.hdf = adcore.NDFileHDFIO(prefix + hdf_suffix) - super().__init__( - AravisController(self.drv, gpio_number=gpio_number), - adcore.ADHDFWriter( - self.hdf, - path_provider, - lambda: self.name, - adcore.ADBaseDatasetDescriber(self.drv), - ), - config_sigs=(self.drv.acquire_time,), + prefix, + path_provider, + adcore.ADHDFWriter, + hdf_suffix, + AravisController, + AravisDriverIO, + drv_suffix=drv_suffix, name=name, + config_sigs=config_sigs, + gpio_number=gpio_number, ) + self.hdf = self._fileio + + @property + def controller(self) -> AravisController: + return cast(AravisController, self._controller) def get_external_trigger_gpio(self): - return self._controller.gpio_number + return self.controller.gpio_number def set_external_trigger_gpio(self, gpio_number: AravisController.GPIO_NUMBER): supported_gpio_numbers = get_args(AravisController.GPIO_NUMBER) @@ -54,8 +55,36 @@ def set_external_trigger_gpio(self, gpio_number: AravisController.GPIO_NUMBER): f"indices: {supported_gpio_numbers} but was asked to " f"use {gpio_number}" ) - self._controller.gpio_number = gpio_number + self.controller.gpio_number = gpio_number - @property - def hints(self) -> Hints: - return self._writer.hints + +class AravisDetectorTIFF(adcore.AreaDetector): + """ + Ophyd-async implementation of an ADAravis Detector. + The detector may be configured for an external trigger on a GPIO port, + which must be done prior to preparing the detector + """ + + def __init__( + self, + prefix: str, + path_provider: PathProvider, + drv_suffix="cam1:", + hdf_suffix="HDF1:", + name="", + gpio_number: AravisController.GPIO_NUMBER = 1, + config_sigs: Sequence[SignalR] = (), + ): + super().__init__( + prefix, + path_provider, + adcore.ADTIFFWriter, + hdf_suffix, + AravisController, + AravisDriverIO, + drv_suffix=drv_suffix, + name=name, + config_sigs=config_sigs, + gpio_number=gpio_number, + ) + self.tiff = self._fileio diff --git a/src/ophyd_async/epics/adaravis/_aravis_controller.py b/src/ophyd_async/epics/adaravis/_aravis_controller.py index 80b826db2d..e5d362e801 100644 --- a/src/ophyd_async/epics/adaravis/_aravis_controller.py +++ b/src/ophyd_async/epics/adaravis/_aravis_controller.py @@ -1,13 +1,10 @@ import asyncio -from typing import Literal +from typing import Literal, cast from ophyd_async.core import ( - DetectorControl, DetectorTrigger, TriggerInfo, - set_and_wait_for_value, ) -from ophyd_async.core._status import AsyncStatus from ophyd_async.epics import adcore from ._aravis_io import AravisDriverIO, AravisTriggerMode, AravisTriggerSource @@ -18,13 +15,16 @@ _HIGHEST_POSSIBLE_DEADTIME = 1961e-6 -class AravisController(DetectorControl): +class AravisController(adcore.ADBaseController): GPIO_NUMBER = Literal[1, 2, 3, 4] def __init__(self, driver: AravisDriverIO, gpio_number: GPIO_NUMBER) -> None: - self._drv = driver + super().__init__(driver) self.gpio_number = gpio_number - self._arm_status: AsyncStatus | None = None + + @property + def driver(self) -> AravisDriverIO: + return cast(AravisDriverIO, self._driver) def get_deadtime(self, exposure: float | None) -> float: return _HIGHEST_POSSIBLE_DEADTIME @@ -35,25 +35,18 @@ async def prepare(self, trigger_info: TriggerInfo): else: image_mode = adcore.ImageMode.multiple if (exposure := trigger_info.livetime) is not None: - await self._drv.acquire_time.set(exposure) + await self.driver.acquire_time.set(exposure) trigger_mode, trigger_source = self._get_trigger_info(trigger_info.trigger) # trigger mode must be set first and on it's own! - await self._drv.trigger_mode.set(trigger_mode) + await self.driver.trigger_mode.set(trigger_mode) await asyncio.gather( - self._drv.trigger_source.set(trigger_source), - self._drv.num_images.set(num), - self._drv.image_mode.set(image_mode), + self.driver.trigger_source.set(trigger_source), + self.driver.num_images.set(num), + self.driver.image_mode.set(image_mode), ) - async def arm(self): - self._arm_status = await set_and_wait_for_value(self._drv.acquire, True) - - async def wait_for_idle(self): - if self._arm_status: - await self._arm_status - def _get_trigger_info( self, trigger: DetectorTrigger ) -> tuple[AravisTriggerMode, AravisTriggerSource]: @@ -72,6 +65,3 @@ def _get_trigger_info( return AravisTriggerMode.off, "Freerun" else: return (AravisTriggerMode.on, f"Line{self.gpio_number}") # type: ignore - - async def disarm(self): - await adcore.stop_busy_record(self._drv.acquire, False, timeout=1) diff --git a/src/ophyd_async/epics/adpilatus/_pilatus.py b/src/ophyd_async/epics/adpilatus/_pilatus.py index 113b780fa2..86c6b44f31 100644 --- a/src/ophyd_async/epics/adpilatus/_pilatus.py +++ b/src/ophyd_async/epics/adpilatus/_pilatus.py @@ -1,8 +1,8 @@ +from collections.abc import Sequence from enum import Enum -from bluesky.protocols import Hints - -from ophyd_async.core import PathProvider, StandardDetector +from ophyd_async.core import PathProvider +from ophyd_async.core._signal import SignalR from ophyd_async.epics import adcore from ._pilatus_controller import PilatusController @@ -23,36 +23,57 @@ class PilatusReadoutTime(float, Enum): pilatus3 = 0.95e-3 -class PilatusDetector(StandardDetector): +class PilatusDetector(adcore.AreaDetector): """A Pilatus StandardDetector writing HDF files""" - _controller: PilatusController - _writer: adcore.ADHDFWriter - def __init__( self, prefix: str, path_provider: PathProvider, - readout_time: PilatusReadoutTime = PilatusReadoutTime.pilatus3, drv_suffix: str = "cam1:", hdf_suffix: str = "HDF1:", name: str = "", + config_sigs: Sequence[SignalR] = (), + readout_time: PilatusReadoutTime = PilatusReadoutTime.pilatus3, ): - self.drv = PilatusDriverIO(prefix + drv_suffix) - self.hdf = adcore.NDFileHDFIO(prefix + hdf_suffix) - super().__init__( - PilatusController(self.drv, readout_time=readout_time.value), - adcore.ADHDFWriter( - self.hdf, - path_provider, - lambda: self.name, - adcore.ADBaseDatasetDescriber(self.drv), - ), - config_sigs=(self.drv.acquire_time,), + prefix, + path_provider, + adcore.ADHDFWriter, + hdf_suffix, + PilatusController, + PilatusDriverIO, + drv_suffix=drv_suffix, name=name, + config_sigs=config_sigs, + readout_time=readout_time, ) + self.hdf = self._fileio - @property - def hints(self) -> Hints: - return self._writer.hints + +class PilatusDetectorTIFF(adcore.AreaDetector): + """A Pilatus StandardDetector writing HDF files""" + + def __init__( + self, + prefix: str, + path_provider: PathProvider, + drv_suffix: str = "cam1:", + tiff_suffix: str = "TIFF1:", + name: str = "", + config_sigs: Sequence[SignalR] = (), + readout_time: PilatusReadoutTime = PilatusReadoutTime.pilatus3, + ): + super().__init__( + prefix, + path_provider, + adcore.ADTIFFWriter, + tiff_suffix, + PilatusController, + PilatusDriverIO, + drv_suffix=drv_suffix, + name=name, + config_sigs=config_sigs, + readout_time=readout_time, + ) + self.tiff = self._fileio diff --git a/src/ophyd_async/epics/adpilatus/_pilatus_controller.py b/src/ophyd_async/epics/adpilatus/_pilatus_controller.py index 9e8bd54aef..7e2fac409a 100644 --- a/src/ophyd_async/epics/adpilatus/_pilatus_controller.py +++ b/src/ophyd_async/epics/adpilatus/_pilatus_controller.py @@ -1,19 +1,18 @@ import asyncio +from typing import cast from ophyd_async.core import ( DEFAULT_TIMEOUT, - DetectorControl, DetectorTrigger, wait_for_value, ) from ophyd_async.core._detector import TriggerInfo -from ophyd_async.core._status import AsyncStatus from ophyd_async.epics import adcore from ._pilatus_io import PilatusDriverIO, PilatusTriggerMode -class PilatusController(DetectorControl): +class PilatusController(adcore.ADBaseController): _supported_trigger_types = { DetectorTrigger.internal: PilatusTriggerMode.internal, DetectorTrigger.constant_gate: PilatusTriggerMode.ext_enable, @@ -25,44 +24,42 @@ def __init__( driver: PilatusDriverIO, readout_time: float, ) -> None: - self._drv = driver + super().__init__(driver) self._readout_time = readout_time - self._arm_status: AsyncStatus | None = None + + @property + def driver(self) -> PilatusDriverIO: + return cast(PilatusDriverIO, self._driver) def get_deadtime(self, exposure: float | None) -> float: return self._readout_time async def prepare(self, trigger_info: TriggerInfo): if trigger_info.livetime is not None: - await adcore.set_exposure_time_and_acquire_period_if_supplied( - self, self._drv, trigger_info.livetime + await self.set_exposure_time_and_acquire_period_if_supplied( + trigger_info.livetime ) await asyncio.gather( - self._drv.trigger_mode.set(self._get_trigger_mode(trigger_info.trigger)), - self._drv.num_images.set( + self.driver.trigger_mode.set(self._get_trigger_mode(trigger_info.trigger)), + self.driver.num_images.set( 999_999 if trigger_info.number == 0 else trigger_info.number ), - self._drv.image_mode.set(adcore.ImageMode.multiple), + self.driver.image_mode.set(adcore.ImageMode.multiple), ) async def arm(self): # Standard arm the detector and wait for the acquire PV to be True - self._arm_status = await adcore.start_acquiring_driver_and_ensure_status( - self._drv - ) + self._arm_status = await self.start_acquiring_driver_and_ensure_status() + # The pilatus has an additional PV that goes True when the camserver # is actually ready. Should wait for that too or we risk dropping # a frame await wait_for_value( - self._drv.armed, + self.driver.armed, True, timeout=DEFAULT_TIMEOUT, ) - async def wait_for_idle(self): - if self._arm_status: - await self._arm_status - @classmethod def _get_trigger_mode(cls, trigger: DetectorTrigger) -> PilatusTriggerMode: if trigger not in cls._supported_trigger_types.keys(): @@ -72,6 +69,3 @@ def _get_trigger_mode(cls, trigger: DetectorTrigger) -> PilatusTriggerMode: f"use {trigger}" ) return cls._supported_trigger_types[trigger] - - async def disarm(self): - await adcore.stop_busy_record(self._drv.acquire, False, timeout=1) From 5b9f60f3ed477231afd84895c7ed7357352e3083 Mon Sep 17 00:00:00 2001 From: Jakub Wlodek Date: Thu, 10 Oct 2024 09:18:56 -0400 Subject: [PATCH 10/60] Update all tests to make sure they still pass with changes --- .../epics/adkinetix/_kinetix_controller.py | 2 +- .../epics/advimba/_vimba_controller.py | 2 +- tests/epics/adcore/test_drivers.py | 48 ++++++++----------- tests/epics/adcore/test_scans.py | 8 ++-- tests/epics/adcore/test_writers.py | 20 +++++++- tests/epics/adpilatus/test_pilatus.py | 13 +++-- tests/epics/adsimdetector/test_sim.py | 37 +++++++++----- tests/epics/conftest.py | 29 +++++++---- 8 files changed, 99 insertions(+), 60 deletions(-) diff --git a/src/ophyd_async/epics/adkinetix/_kinetix_controller.py b/src/ophyd_async/epics/adkinetix/_kinetix_controller.py index bcf26e85a6..2622f3d548 100644 --- a/src/ophyd_async/epics/adkinetix/_kinetix_controller.py +++ b/src/ophyd_async/epics/adkinetix/_kinetix_controller.py @@ -27,7 +27,7 @@ def driver(self) -> KinetixDriverIO: return cast(KinetixDriverIO, self._driver) def get_deadtime(self, exposure: float | None) -> float: - return 0.0000001 + return 0.001 async def prepare(self, trigger_info: TriggerInfo): await asyncio.gather( diff --git a/src/ophyd_async/epics/advimba/_vimba_controller.py b/src/ophyd_async/epics/advimba/_vimba_controller.py index afd103bdbc..379f0ec5d6 100644 --- a/src/ophyd_async/epics/advimba/_vimba_controller.py +++ b/src/ophyd_async/epics/advimba/_vimba_controller.py @@ -34,7 +34,7 @@ def driver(self) -> VimbaDriverIO: return cast(VimbaDriverIO, self._driver) def get_deadtime(self, exposure: float | None) -> float: - return 0.00001 + return 0.001 async def prepare(self, trigger_info: TriggerInfo): await asyncio.gather( diff --git a/tests/epics/adcore/test_drivers.py b/tests/epics/adcore/test_drivers.py index 379eac9a0f..86a5aba144 100644 --- a/tests/epics/adcore/test_drivers.py +++ b/tests/epics/adcore/test_drivers.py @@ -1,10 +1,8 @@ import asyncio -from unittest.mock import Mock import pytest from ophyd_async.core import ( - DetectorControl, DeviceCollector, get_mock_put, set_mock_value, @@ -22,21 +20,20 @@ def driver(RE) -> adcore.ADBaseIO: @pytest.fixture -async def controller(RE, driver: adcore.ADBaseIO) -> Mock: - controller = Mock(spec=DetectorControl) - controller.get_deadtime.return_value = TEST_DEADTIME +async def controller(RE, driver: adcore.ADBaseIO) -> adcore.ADBaseController: + controller = adcore.ADBaseController(driver) + controller.get_deadtime = lambda exposure: TEST_DEADTIME return controller async def test_set_exposure_time_and_acquire_period_if_supplied_is_a_noop_if_no_exposure_supplied( # noqa: E501 - controller: DetectorControl, + controller: adcore.ADBaseController, driver: adcore.ADBaseIO, ): put_exposure = get_mock_put(driver.acquire_time) put_acquire_period = get_mock_put(driver.acquire_period) - await adcore.set_exposure_time_and_acquire_period_if_supplied( - controller, driver, None - ) + await controller.set_exposure_time_and_acquire_period_if_supplied(None) + put_exposure.assert_not_called() put_acquire_period.assert_not_called() @@ -50,49 +47,46 @@ async def test_set_exposure_time_and_acquire_period_if_supplied_is_a_noop_if_no_ ], ) async def test_set_exposure_time_and_acquire_period_if_supplied_uses_deadtime( - controller: DetectorControl, - driver: adcore.ADBaseIO, + controller: adcore.ADBaseController, exposure: float, expected_exposure: float, expected_acquire_period: float, ): - await adcore.set_exposure_time_and_acquire_period_if_supplied( - controller, driver, exposure - ) - actual_exposure = await driver.acquire_time.get_value() - actual_acquire_period = await driver.acquire_period.get_value() + await controller.set_exposure_time_and_acquire_period_if_supplied(exposure) + actual_exposure = await controller.driver.acquire_time.get_value() + actual_acquire_period = await controller.driver.acquire_period.get_value() assert expected_exposure == actual_exposure assert expected_acquire_period == actual_acquire_period async def test_start_acquiring_driver_and_ensure_status_flags_immediate_failure( - driver: adcore.ADBaseIO, + controller: adcore.ADBaseController, ): - set_mock_value(driver.detector_state, adcore.DetectorState.Error) - acquiring = await adcore.start_acquiring_driver_and_ensure_status( - driver, timeout=0.01 - ) + set_mock_value(controller.driver.detector_state, adcore.DetectorState.Error) + acquiring = await controller.start_acquiring_driver_and_ensure_status() with pytest.raises(ValueError): await acquiring async def test_start_acquiring_driver_and_ensure_status_fails_after_some_time( - driver: adcore.ADBaseIO, + controller: adcore.ADBaseController, ): """This test ensures a failing status is captured halfway through acquisition. Real world application; it takes some time to start acquiring, and during that time the detector gets itself into a bad state. """ - set_mock_value(driver.detector_state, adcore.DetectorState.Idle) + set_mock_value(controller.driver.detector_state, adcore.DetectorState.Idle) async def wait_then_fail(): await asyncio.sleep(0) - set_mock_value(driver.detector_state, adcore.DetectorState.Disconnected) + set_mock_value( + controller.driver.detector_state, adcore.DetectorState.Disconnected + ) + + controller.frame_timeout = 0.1 - acquiring = await adcore.start_acquiring_driver_and_ensure_status( - driver, timeout=0.1 - ) + acquiring = await controller.start_acquiring_driver_and_ensure_status() await wait_then_fail() with pytest.raises(ValueError): diff --git a/tests/epics/adcore/test_scans.py b/tests/epics/adcore/test_scans.py index b6f9d10f11..e125ed5474 100644 --- a/tests/epics/adcore/test_scans.py +++ b/tests/epics/adcore/test_scans.py @@ -19,7 +19,7 @@ TriggerLogic, set_mock_value, ) -from ophyd_async.epics import adcore, adsimdetector +from ophyd_async.epics import adcore class DummyTriggerLogic(TriggerLogic[int]): @@ -53,11 +53,11 @@ def get_deadtime(self, exposure: float | None) -> float: @pytest.fixture -def controller(RE) -> adsimdetector.SimController: +def controller(RE) -> adcore.ADBaseController: with DeviceCollector(mock=True): drv = adcore.ADBaseIO("DRV") - return adsimdetector.SimController(drv) + return adcore.ADBaseController(drv) @pytest.fixture @@ -77,7 +77,7 @@ def writer(RE, static_path_provider, tmp_path: Path) -> adcore.ADHDFWriter: async def test_hdf_writer_fails_on_timeout_with_stepscan( RE: RunEngine, writer: adcore.ADHDFWriter, - controller: adsimdetector.SimController, + controller: adcore.ADBaseController, ): set_mock_value(writer.hdf.file_path_exists, True) detector: StandardDetector[Any] = StandardDetector( diff --git a/tests/epics/adcore/test_writers.py b/tests/epics/adcore/test_writers.py index af32f86667..7c5b7edb4e 100644 --- a/tests/epics/adcore/test_writers.py +++ b/tests/epics/adcore/test_writers.py @@ -38,6 +38,18 @@ async def hdf_writer( ) +@pytest.fixture +async def tiff_writer( + RE, static_path_provider: StaticPathProvider +) -> adcore.ADTIFFWriter: + async with DeviceCollector(mock=True): + tiff = adcore.NDFileIO("TIFF:") + + return adcore.ADTIFFWriter( + tiff, static_path_provider, lambda: "test", DummyDatasetDescriber() + ) + + @pytest.fixture async def hdf_writer_with_stats( RE, static_path_provider: StaticPathProvider @@ -71,13 +83,19 @@ async def detectors( return detectors -async def test_collect_stream_docs(hdf_writer: adcore.ADHDFWriter): +async def test_hdf_writer_collect_stream_docs(hdf_writer: adcore.ADHDFWriter): assert hdf_writer._file is None [item async for item in hdf_writer.collect_stream_docs(1)] assert hdf_writer._file +async def test_tiff_writer_collect_stream_docs(tiff_writer: adcore.ADTIFFWriter): + assert tiff_writer._emitted_resource is None + [item async for item in tiff_writer.collect_stream_docs(1)] + assert tiff_writer._emitted_resource + + async def test_stats_describe_when_plugin_configured( hdf_writer_with_stats: adcore.ADHDFWriter, ): diff --git a/tests/epics/adpilatus/test_pilatus.py b/tests/epics/adpilatus/test_pilatus.py index 192c466a13..6011885096 100644 --- a/tests/epics/adpilatus/test_pilatus.py +++ b/tests/epics/adpilatus/test_pilatus.py @@ -1,5 +1,6 @@ import asyncio from collections.abc import Awaitable, Callable +from typing import cast from unittest.mock import patch import pytest @@ -10,6 +11,7 @@ set_mock_value, ) from ophyd_async.epics import adcore, adpilatus +from ophyd_async.epics.adpilatus import PilatusController, PilatusDriverIO @pytest.fixture @@ -18,7 +20,7 @@ def test_adpilatus(ad_standard_det_factory) -> adpilatus.PilatusDetector: async def test_deadtime_overridable(test_adpilatus: adpilatus.PilatusDetector): - pilatus_controller = test_adpilatus._controller + pilatus_controller = cast(PilatusController, test_adpilatus.controller) pilatus_controller._readout_time = adpilatus.PilatusReadoutTime.pilatus2 # deadtime invariant with exposure time @@ -85,15 +87,16 @@ async def _trigger( expected_trigger_mode: adpilatus.PilatusTriggerMode, trigger_and_complete: Callable[[], Awaitable], ): + pilatus_driver = cast(PilatusDriverIO, test_adpilatus.drv) # Default TriggerMode assert ( - await test_adpilatus.drv.trigger_mode.get_value() + await pilatus_driver.trigger_mode.get_value() ) == adpilatus.PilatusTriggerMode.internal await trigger_and_complete() # TriggerSource changes - assert (await test_adpilatus.drv.trigger_mode.get_value()) == expected_trigger_mode + assert (await pilatus_driver.trigger_mode.get_value()) == expected_trigger_mode async def test_hints_from_hdf_writer(test_adpilatus: adpilatus.PilatusDetector): @@ -134,8 +137,8 @@ async def dummy_open(multiplier: int = 0): async def test_pilatus_controller(test_adpilatus: adpilatus.PilatusDetector): - pilatus = test_adpilatus._controller - pilatus_driver = pilatus._drv + pilatus = test_adpilatus.controller + pilatus_driver = cast(PilatusDriverIO, pilatus.driver) set_mock_value(pilatus_driver.armed, True) await pilatus.prepare(TriggerInfo(number=1, trigger=DetectorTrigger.constant_gate)) await pilatus.arm() diff --git a/tests/epics/adsimdetector/test_sim.py b/tests/epics/adsimdetector/test_sim.py index 637c3c6ed7..15803b64a5 100644 --- a/tests/epics/adsimdetector/test_sim.py +++ b/tests/epics/adsimdetector/test_sim.py @@ -2,6 +2,7 @@ import time from collections import defaultdict +from collections.abc import Callable, Sequence from pathlib import Path from typing import cast from unittest.mock import patch @@ -29,6 +30,11 @@ def test_adsimdetector(ad_standard_det_factory): return ad_standard_det_factory(adsimdetector.SimDetector) +@pytest.fixture +def test_adsimdetector_tiff(ad_standard_det_factory): + return ad_standard_det_factory(adsimdetector.SimDetectorTIFF) + + @pytest.fixture def two_test_adsimdetectors(ad_standard_det_factory): deta = ad_standard_det_factory(adsimdetector.SimDetector) @@ -37,7 +43,7 @@ def two_test_adsimdetectors(ad_standard_det_factory): return deta, detb -def count_sim(dets: list[adsimdetector.SimDetector], times: int = 1): +def count_sim(dets: Sequence[adcore.AreaDetector], times: int = 1): """Test plan to do the equivalent of bp.count for a sim detector.""" yield from bps.stage_all(*dets) @@ -46,7 +52,7 @@ def count_sim(dets: list[adsimdetector.SimDetector], times: int = 1): read_values = {} for det in dets: read_values[det] = yield from bps.rd( - cast(adcore.ADHDFWriter, det.writer).hdf.num_captured + cast(adcore.ADWriter, det.writer).fileio.num_captured ) for det in dets: @@ -55,7 +61,7 @@ def count_sim(dets: list[adsimdetector.SimDetector], times: int = 1): yield from bps.sleep(0.2) [ set_mock_value( - cast(adcore.ADHDFWriter, det.writer).hdf.num_captured, + cast(adcore.ADWriter, det.writer).fileio.num_captured, read_values[det] + 1, ) for det in dets @@ -150,9 +156,7 @@ async def test_two_detectors_step( for det in two_test_adsimdetectors ] - controller_a = cast( - adsimdetector.SimController, two_test_adsimdetectors[0].controller - ) + controller_a = cast(adcore.ADBaseController, two_test_adsimdetectors[0].controller) writer_a = cast(adcore.ADHDFWriter, two_test_adsimdetectors[0].writer) writer_b = cast(adcore.ADHDFWriter, two_test_adsimdetectors[1].writer) info_a = writer_a._path_provider(device_name=writer_a._name_provider()) @@ -217,26 +221,37 @@ def plan(): assert event["data"] == {} +@pytest.mark.parametrize( + "detector_class", [adsimdetector.SimDetector, adsimdetector.SimDetectorTIFF] +) async def test_detector_writes_to_file( - RE: RunEngine, test_adsimdetector: adsimdetector.SimDetector, tmp_path: Path + RE: RunEngine, + ad_standard_det_factory: Callable, + detector_class: type[adsimdetector.SimDetector], + tmp_path: Path, ): + test_adsimdetector = ad_standard_det_factory(detector_class) + names = [] docs = [] RE.subscribe(lambda name, _: names.append(name)) RE.subscribe(lambda _, doc: docs.append(doc)) set_mock_value( - cast(adcore.ADHDFWriter, test_adsimdetector._writer).hdf.file_path_exists, True + cast(adcore.ADHDFWriter, test_adsimdetector.writer).fileio.file_path_exists, + True, ) RE(count_sim([test_adsimdetector], times=3)) assert await cast( - adcore.ADHDFWriter, test_adsimdetector.writer - ).hdf.file_path.get_value() == str(tmp_path) + adcore.ADWriter, test_adsimdetector.writer + ).fileio.file_path.get_value() == str(tmp_path) descriptor_index = names.index("descriptor") - assert docs[descriptor_index].get("data_keys").get("test_adsim1").get("shape") == ( + assert docs[descriptor_index].get("data_keys").get(test_adsimdetector.name).get( + "shape" + ) == ( 10, 10, ) diff --git a/tests/epics/conftest.py b/tests/epics/conftest.py index daf11fee18..6b8d5772be 100644 --- a/tests/epics/conftest.py +++ b/tests/epics/conftest.py @@ -1,26 +1,33 @@ import os +from builtins import float, len, type from collections.abc import Callable import pytest from bluesky.run_engine import RunEngine -from ophyd_async.core._detector import StandardDetector from ophyd_async.core._device import DeviceCollector from ophyd_async.core._mock_signal_utils import callback_on_mock_put, set_mock_value +from ophyd_async.epics import adcore @pytest.fixture def ad_standard_det_factory( RE: RunEngine, static_path_provider, -) -> Callable[[StandardDetector, int], StandardDetector]: +) -> Callable[[type[adcore.AreaDetector], int], adcore.AreaDetector]: def generate_ad_standard_det( - ad_standard_detector_class, number=1 - ) -> StandardDetector: + ad_standard_detector_class: type[adcore.AreaDetector], number=1 + ) -> adcore.AreaDetector: # Dynamically generate a name based on the class of detector detector_name = ad_standard_detector_class.__name__ if detector_name.endswith("Detector"): detector_name = detector_name[: -len("Detector")] + elif detector_name.endswith("DetectorTIFF"): + detector_name = ( + detector_name.split("Detector")[0] + + "_" + + detector_name.split("Detector")[1] + ) with DeviceCollector(mock=True): test_adstandard_det = ad_standard_detector_class( @@ -31,26 +38,28 @@ def generate_ad_standard_det( def on_set_file_path_callback(value, **kwargs): if os.path.exists(value): - set_mock_value(test_adstandard_det.hdf.file_path_exists, True) + set_mock_value(test_adstandard_det.writer.fileio.file_path_exists, True) set_mock_value( - test_adstandard_det.hdf.full_file_name, - f"{value}/{static_path_provider._filename_provider(device_name=test_adstandard_det.name)}.h5", + test_adstandard_det.writer.fileio.full_file_name, + f"{value}/{static_path_provider._filename_provider(device_name=test_adstandard_det.name)}{test_adstandard_det.writer._file_extension}", ) callback_on_mock_put( - test_adstandard_det.hdf.file_path, on_set_file_path_callback + test_adstandard_det.writer.fileio.file_path, on_set_file_path_callback ) # Set some sensible defaults to mimic a real detector setup set_mock_value(test_adstandard_det.drv.acquire_time, (number - 0.2)) set_mock_value(test_adstandard_det.drv.acquire_period, float(number)) - set_mock_value(test_adstandard_det.hdf.capture, True) + set_mock_value(test_adstandard_det.writer.fileio.capture, True) # Set number of frames per chunk and frame dimensions to something reasonable - set_mock_value(test_adstandard_det.hdf.num_frames_chunks, 1) set_mock_value(test_adstandard_det.drv.array_size_x, (9 + number)) set_mock_value(test_adstandard_det.drv.array_size_y, (9 + number)) + if isinstance(test_adstandard_det.writer, adcore.ADHDFWriter): + set_mock_value(test_adstandard_det.writer.hdf.num_frames_chunks, 1) + return test_adstandard_det return generate_ad_standard_det From 8bbfd0e061092ac36c734f37fac4107c4d32aeff Mon Sep 17 00:00:00 2001 From: Jakub Wlodek Date: Thu, 10 Oct 2024 09:26:16 -0400 Subject: [PATCH 11/60] Some cleanup --- src/ophyd_async/epics/adcore/_core_logic.py | 3 ++- tests/epics/conftest.py | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ophyd_async/epics/adcore/_core_logic.py b/src/ophyd_async/epics/adcore/_core_logic.py index e3a075eca7..0fb59657b9 100644 --- a/src/ophyd_async/epics/adcore/_core_logic.py +++ b/src/ophyd_async/epics/adcore/_core_logic.py @@ -5,9 +5,10 @@ AsyncStatus, DatasetDescriber, DetectorControl, + DetectorTrigger, + TriggerInfo, set_and_wait_for_value, ) -from ophyd_async.core._detector import DetectorTrigger, TriggerInfo from ._core_io import ADBaseIO, DetectorState from ._utils import ImageMode, convert_ad_dtype_to_np, stop_busy_record diff --git a/tests/epics/conftest.py b/tests/epics/conftest.py index 6b8d5772be..a7cf0f05a6 100644 --- a/tests/epics/conftest.py +++ b/tests/epics/conftest.py @@ -1,5 +1,4 @@ import os -from builtins import float, len, type from collections.abc import Callable import pytest From f6825b42c6e1b870438fb335deb5853a013a0578 Mon Sep 17 00:00:00 2001 From: Jakub Wlodek Date: Fri, 22 Nov 2024 13:41:04 -0500 Subject: [PATCH 12/60] Changes to standard detector to account for controller/writer types in typing --- src/ophyd_async/core/__init__.py | 2 + src/ophyd_async/core/_detector.py | 69 ++++++++++++++++++------------- src/ophyd_async/core/_flyer.py | 4 +- 3 files changed, 44 insertions(+), 31 deletions(-) diff --git a/src/ophyd_async/core/__init__.py b/src/ophyd_async/core/__init__.py index ddd3ab1a80..6c56e8bcdd 100644 --- a/src/ophyd_async/core/__init__.py +++ b/src/ophyd_async/core/__init__.py @@ -1,5 +1,6 @@ from ._detector import ( DetectorController, + DetectorControllerT, DetectorTrigger, DetectorWriter, StandardDetector, @@ -86,6 +87,7 @@ __all__ = [ "DetectorController", + "DetectorControllerT", "DetectorTrigger", "DetectorWriter", "StandardDetector", diff --git a/src/ophyd_async/core/_detector.py b/src/ophyd_async/core/_detector.py index 4bfed6b451..26b46ec3e5 100644 --- a/src/ophyd_async/core/_detector.py +++ b/src/ophyd_async/core/_detector.py @@ -7,12 +7,15 @@ from enum import Enum from functools import cached_property from typing import ( + Any, Generic, + TypeVar, ) from bluesky.protocols import ( Collectable, Flyable, + Hints, Preparable, Reading, Stageable, @@ -27,7 +30,7 @@ from ._protocol import AsyncConfigurable, AsyncReadable from ._signal import SignalR from ._status import AsyncStatus, WatchableAsyncStatus -from ._utils import DEFAULT_TIMEOUT, T, WatcherUpdate, merge_gathered_dicts +from ._utils import DEFAULT_TIMEOUT, WatcherUpdate, merge_gathered_dicts class DetectorTrigger(str, Enum): @@ -91,7 +94,7 @@ def get_deadtime(self, exposure: float | None) -> float: """For a given exposure, how long should the time between exposures be""" @abstractmethod - async def prepare(self, trigger_info: TriggerInfo): + async def prepare(self, trigger_info: TriggerInfo) -> Any: """ Do all necessary steps to prepare the detector for triggers. @@ -161,6 +164,16 @@ def collect_stream_docs(self, indices_written: int) -> AsyncIterator[StreamAsset async def close(self) -> None: """Close writer, blocks until I/O is complete""" + @property + def hints(self) -> Hints: + return {} + + +# Add type vars for controller/type, so we can define +# StandardDetector[KinetixController, ADTIFFWriter] for example +DetectorControllerT = TypeVar("DetectorControllerT", bound=DetectorController) +DetectorWriterT = TypeVar("DetectorWriterT", bound=DetectorWriter) + class StandardDetector( Device, @@ -172,7 +185,7 @@ class StandardDetector( Flyable, Collectable, WritesStreamAssets, - Generic[T], + Generic[DetectorControllerT, DetectorWriterT], ): """ Useful detector base class for step and fly scanning detectors. @@ -181,8 +194,8 @@ class StandardDetector( def __init__( self, - controller: DetectorController, - writer: DetectorWriter, + controller: DetectorControllerT, + writer: DetectorWriterT, config_sigs: Sequence[SignalR] = (), name: str = "", ) -> None: @@ -216,19 +229,11 @@ def __init__( super().__init__(name) - @property - def controller(self) -> DetectorController: - return self._controller - - @property - def writer(self) -> DetectorWriter: - return self._writer - @AsyncStatus.wrap async def stage(self) -> None: # Disarm the detector, stop file writing. await self._check_config_sigs() - await asyncio.gather(self.writer.close(), self.controller.disarm()) + await asyncio.gather(self._writer.close(), self._controller.disarm()) self._trigger_info = None async def _check_config_sigs(self): @@ -249,7 +254,7 @@ async def _check_config_sigs(self): @AsyncStatus.wrap async def unstage(self) -> None: # Stop data writing. - await asyncio.gather(self.writer.close(), self.controller.disarm()) + await asyncio.gather(self._writer.close(), self._controller.disarm()) async def read_configuration(self) -> dict[str, Reading]: return await merge_gathered_dicts(sig.read() for sig in self._config_sigs) @@ -266,6 +271,7 @@ async def describe(self) -> dict[str, DataKey]: @AsyncStatus.wrap async def trigger(self) -> None: + print("In trigger") if self._trigger_info is None: await self.prepare( TriggerInfo( @@ -276,15 +282,16 @@ async def trigger(self) -> None: frame_timeout=None, ) ) + print(self._trigger_info) assert self._trigger_info assert self._trigger_info.trigger is DetectorTrigger.internal # Arm the detector and wait for it to finish. - indices_written = await self.writer.get_indices_written() - await self.controller.arm() - await self.controller.wait_for_idle() + indices_written = await self._writer.get_indices_written() + await self._controller.arm() + await self._controller.wait_for_idle() end_observation = indices_written + 1 - async for index in self.writer.observe_indices_written( + async for index in self._writer.observe_indices_written( DEFAULT_TIMEOUT + (self._trigger_info.livetime or 0) + (self._trigger_info.deadtime or 0) @@ -313,9 +320,9 @@ async def prepare(self, value: TriggerInfo) -> None: value.deadtime ), "Deadtime must be supplied when in externally triggered mode" if value.deadtime: - required = self.controller.get_deadtime(value.livetime) + required = self._controller.get_deadtime(value.livetime) assert required <= value.deadtime, ( - f"Detector {self.controller} needs at least {required}s deadtime, " + f"Detector {self._controller} needs at least {required}s deadtime, " f"but trigger logic provides only {value.deadtime}s" ) self._trigger_info = value @@ -324,12 +331,12 @@ async def prepare(self, value: TriggerInfo) -> None: if isinstance(self._trigger_info.number_of_triggers, list) else [self._trigger_info.number_of_triggers] ) - self._initial_frame = await self.writer.get_indices_written() + self._initial_frame = await self._writer.get_indices_written() self._describe, _ = await asyncio.gather( - self.writer.open(value.multiplier), self.controller.prepare(value) + self._writer.open(value.multiplier), self._controller.prepare(value) ) if value.trigger != DetectorTrigger.internal: - await self.controller.arm() + await self._controller.arm() self._fly_start = time.monotonic() @AsyncStatus.wrap @@ -348,7 +355,7 @@ async def kickoff(self): @WatchableAsyncStatus.wrap async def complete(self): assert self._trigger_info - indices_written = self.writer.observe_indices_written( + indices_written = self._writer.observe_indices_written( self._trigger_info.frame_timeout or ( DEFAULT_TIMEOUT @@ -377,7 +384,7 @@ async def complete(self): self._completable_frames = 0 self._frames_to_complete = 0 self._number_of_triggers_iter = None - await self.controller.wait_for_idle() + await self._controller.wait_for_idle() async def describe_collect(self) -> dict[str, DataKey]: return self._describe @@ -389,9 +396,13 @@ async def collect_asset_docs( # The index is optional, and provided for fly scans, however this needs to be # retrieved for step scans. if index is None: - index = await self.writer.get_indices_written() - async for doc in self.writer.collect_stream_docs(index): + index = await self._writer.get_indices_written() + async for doc in self._writer.collect_stream_docs(index): yield doc async def get_index(self) -> int: - return await self.writer.get_indices_written() + return await self._writer.get_indices_written() + + @property + def hints(self) -> Hints: + return self._writer.hints diff --git a/src/ophyd_async/core/_flyer.py b/src/ophyd_async/core/_flyer.py index 3e54b87705..91e8efb17f 100644 --- a/src/ophyd_async/core/_flyer.py +++ b/src/ophyd_async/core/_flyer.py @@ -1,5 +1,5 @@ from abc import ABC, abstractmethod -from typing import Generic +from typing import Any, Generic from bluesky.protocols import Flyable, Preparable, Stageable @@ -10,7 +10,7 @@ class FlyerController(ABC, Generic[T]): @abstractmethod - async def prepare(self, value: T): + async def prepare(self, value: T) -> Any: """Move to the start of the flyscan""" @abstractmethod From 651b80d1e4cc2e1d9c1df813c0bb85f2b857c36b Mon Sep 17 00:00:00 2001 From: Jakub Wlodek Date: Fri, 22 Nov 2024 13:50:16 -0500 Subject: [PATCH 13/60] Significant changes to base detector, controller, and writer classes --- src/ophyd_async/epics/adcore/__init__.py | 7 +- .../epics/adcore/_core_detector.py | 84 ++++++-------- src/ophyd_async/epics/adcore/_core_io.py | 19 +++- src/ophyd_async/epics/adcore/_core_logic.py | 44 ++++---- src/ophyd_async/epics/adcore/_core_writer.py | 81 ++++++++++---- src/ophyd_async/epics/adcore/_hdf_writer.py | 104 +++++++++--------- .../epics/adcore/_single_trigger.py | 7 +- src/ophyd_async/epics/adcore/_tiff_writer.py | 18 +-- 8 files changed, 205 insertions(+), 159 deletions(-) diff --git a/src/ophyd_async/epics/adcore/__init__.py b/src/ophyd_async/epics/adcore/__init__.py index 6dd0f3226d..16587efa01 100644 --- a/src/ophyd_async/epics/adcore/__init__.py +++ b/src/ophyd_async/epics/adcore/__init__.py @@ -1,13 +1,15 @@ from ._core_detector import AreaDetector from ._core_io import ( + ADBaseDatasetDescriber, ADBaseIO, DetectorState, NDArrayBaseIO, NDFileHDFIO, NDFileIO, + NDPluginBaseIO, NDPluginStatsIO, ) -from ._core_logic import DEFAULT_GOOD_STATES, ADBaseController, ADBaseDatasetDescriber +from ._core_logic import DEFAULT_GOOD_STATES, ADBaseController from ._core_writer import ADWriter from ._hdf_writer import ADHDFWriter from ._single_trigger import SingleTriggerDetector @@ -25,15 +27,16 @@ __all__ = [ "ADBaseIO", + "AreaDetector", "DetectorState", "NDArrayBaseIO", "NDFileIO", "NDFileHDFIO", + "NDPluginBaseIO", "NDPluginStatsIO", "DEFAULT_GOOD_STATES", "ADBaseDatasetDescriber", "ADBaseController", - "AreaDetector", "ADWriter", "ADHDFWriter", "ADTIFFWriter", diff --git a/src/ophyd_async/epics/adcore/_core_detector.py b/src/ophyd_async/epics/adcore/_core_detector.py index 23e3c0b126..a27971fe7b 100644 --- a/src/ophyd_async/epics/adcore/_core_detector.py +++ b/src/ophyd_async/epics/adcore/_core_detector.py @@ -1,66 +1,52 @@ from collections.abc import Sequence -from typing import cast - -from bluesky.protocols import HasHints, Hints from ophyd_async.core import PathProvider, SignalR, StandardDetector -from ._core_io import ADBaseIO, NDFileHDFIO, NDFileIO -from ._core_logic import ADBaseController, ADBaseDatasetDescriber -from ._core_writer import ADWriter -from ._hdf_writer import ADHDFWriter -from ._tiff_writer import ADTIFFWriter - - -def get_io_class_for_writer(writer_class: type[ADWriter]): - writer_to_io_map = { - ADWriter: NDFileIO, - ADHDFWriter: NDFileHDFIO, - ADTIFFWriter: NDFileIO, - } - return writer_to_io_map[writer_class] +from ._core_io import ADBaseDatasetDescriber, ADBaseIO, NDPluginBaseIO +from ._core_logic import ADBaseControllerT +from ._core_writer import ADWriterT -class AreaDetector(StandardDetector, HasHints): - _controller: ADBaseController - _writer: ADWriter - +class AreaDetector(StandardDetector[ADBaseControllerT, ADWriterT]): def __init__( self, prefix: str, + driver: ADBaseIO, + controller: ADBaseControllerT, + writer_cls: type[ADWriterT], path_provider: PathProvider, - writer_class: type[ADWriter] = ADWriter, - writer_suffix: str = "", - controller_class: type[ADBaseController] = ADBaseController, - drv_class: type[ADBaseIO] = ADBaseIO, - drv_suffix: str = "cam1:", - name: str = "", + plugins: dict[str, NDPluginBaseIO] | None, config_sigs: Sequence[SignalR] = (), - **kwargs, + name: str = "", + fileio_suffix: str | None = None, ): - self.drv = drv_class(prefix + drv_suffix) - self._fileio = get_io_class_for_writer(writer_class)(prefix + writer_suffix) - - super().__init__( - controller_class(self.drv, **kwargs), - writer_class( - self._fileio, - path_provider, - lambda: self.name, - ADBaseDatasetDescriber(self.drv), - ), - config_sigs=(self.drv.acquire_period, self.drv.acquire_time, *config_sigs), + self.drv = driver + writer, self.fileio = writer_cls.writer_and_io( + prefix + (fileio_suffix or writer_cls.default_suffix), + path_provider, + lambda: name, + ADBaseDatasetDescriber(self.drv), + plugins=plugins, name=name, ) - @property - def controller(self) -> ADBaseController: - return cast(ADBaseController, self._controller) + if plugins is not None: + for name, plugin in plugins.items(): + setattr(self, name, plugin) - @property - def writer(self) -> ADWriter: - return cast(ADWriter, self._writer) + super().__init__( + controller, + writer, + (self.drv.acquire_period, self.drv.acquire_time, *config_sigs), + name=name, + ) - @property - def hints(self) -> Hints: - return self._writer.hints + def get_plugin( + self, name: str, plugin_type: type[NDPluginBaseIO] = NDPluginBaseIO + ) -> NDPluginBaseIO: + plugin = getattr(self, name, None) + if not isinstance(plugin, plugin_type): + raise TypeError( + f"Expected {self.name}.{name} to be a {plugin_type}, got {plugin}" + ) + return plugin diff --git a/src/ophyd_async/epics/adcore/_core_io.py b/src/ophyd_async/epics/adcore/_core_io.py index 9d21d37e89..2ccd4face0 100644 --- a/src/ophyd_async/epics/adcore/_core_io.py +++ b/src/ophyd_async/epics/adcore/_core_io.py @@ -1,13 +1,15 @@ +import asyncio from enum import Enum from ophyd_async.core import Device +from ophyd_async.core._providers import DatasetDescriber from ophyd_async.epics.signal import ( epics_signal_r, epics_signal_rw, epics_signal_rw_rbv, ) -from ._utils import ADBaseDataType, FileWriteMode, ImageMode +from ._utils import ADBaseDataType, FileWriteMode, ImageMode, convert_ad_dtype_to_np class NDArrayBaseIO(Device): @@ -24,6 +26,21 @@ def __init__(self, prefix: str, name: str = "") -> None: super().__init__(name=name) +class ADBaseDatasetDescriber(DatasetDescriber): + def __init__(self, driver: NDArrayBaseIO) -> None: + self._driver = driver + + async def np_datatype(self) -> str: + return convert_ad_dtype_to_np(await self._driver.data_type.get_value()) + + async def shape(self) -> tuple[int, int]: + shape = await asyncio.gather( + self._driver.array_size_y.get_value(), + self._driver.array_size_x.get_value(), + ) + return shape + + class NDPluginBaseIO(NDArrayBaseIO): def __init__(self, prefix: str, name: str = "") -> None: self.nd_array_port = epics_signal_rw_rbv(str, prefix + "NDArrayPort") diff --git a/src/ophyd_async/epics/adcore/_core_logic.py b/src/ophyd_async/epics/adcore/_core_logic.py index 9f045b03c2..dbd37e0f2d 100644 --- a/src/ophyd_async/epics/adcore/_core_logic.py +++ b/src/ophyd_async/epics/adcore/_core_logic.py @@ -1,9 +1,9 @@ import asyncio +from typing import Any, Generic, TypeVar, get_args from ophyd_async.core import ( DEFAULT_TIMEOUT, AsyncStatus, - DatasetDescriber, DetectorController, DetectorTrigger, TriggerInfo, @@ -11,7 +11,7 @@ ) from ._core_io import ADBaseIO, DetectorState -from ._utils import ImageMode, convert_ad_dtype_to_np, stop_busy_record +from ._utils import ImageMode, stop_busy_record # Default set of states that we should consider "good" i.e. the acquisition # is complete and went well @@ -19,26 +19,14 @@ [DetectorState.Idle, DetectorState.Aborted] ) +ADBaseIOT = TypeVar("ADBaseIOT", bound=ADBaseIO) +ADBaseControllerT = TypeVar("ADBaseControllerT", bound="ADBaseController") -class ADBaseDatasetDescriber(DatasetDescriber): - def __init__(self, driver: ADBaseIO) -> None: - self._driver = driver - - async def np_datatype(self) -> str: - return convert_ad_dtype_to_np(await self._driver.data_type.get_value()) - - async def shape(self) -> tuple[int, int]: - shape = await asyncio.gather( - self._driver.array_size_y.get_value(), - self._driver.array_size_x.get_value(), - ) - return shape - -class ADBaseController(DetectorController): +class ADBaseController(DetectorController, Generic[ADBaseIOT]): def __init__( self, - driver: ADBaseIO, + driver: ADBaseIOT, good_states: frozenset[DetectorState] = DEFAULT_GOOD_STATES, ) -> None: self._driver = driver @@ -46,14 +34,26 @@ def __init__( self.frame_timeout = DEFAULT_TIMEOUT self._arm_status: AsyncStatus | None = None - @property - def driver(self) -> ADBaseIO: - return self._driver + @classmethod + def controller_and_drv( + cls: type[ADBaseControllerT], + prefix: str, + good_states: frozenset[DetectorState] = DEFAULT_GOOD_STATES, + name: str = "", + ) -> tuple[ADBaseControllerT, ADBaseIOT]: + try: + driver_cls = get_args(cls.__orig_bases__[0])[0] # type: ignore + except IndexError as err: + raise RuntimeError("Driver IO class for controller not specified!") from err + + driver = driver_cls(prefix, name=name) + controller = cls(driver, good_states=good_states) + return controller, driver def get_deadtime(self, exposure: float | None) -> float: return 0.002 - async def prepare(self, trigger_info: TriggerInfo): + async def prepare(self, trigger_info: TriggerInfo) -> Any: assert ( trigger_info.trigger == DetectorTrigger.internal ), "fly scanning (i.e. external triggering) is not supported for this device" diff --git a/src/ophyd_async/epics/adcore/_core_writer.py b/src/ophyd_async/epics/adcore/_core_writer.py index 08483156e0..3e73fe29d9 100644 --- a/src/ophyd_async/epics/adcore/_core_writer.py +++ b/src/ophyd_async/epics/adcore/_core_writer.py @@ -1,6 +1,8 @@ import asyncio from collections.abc import AsyncGenerator, AsyncIterator +from enum import Enum from pathlib import Path +from typing import Generic, TypeVar, get_args from urllib.parse import urlunparse from bluesky.protocols import Hints, StreamAsset @@ -20,21 +22,35 @@ from ophyd_async.core._status import AsyncStatus from ophyd_async.core._utils import DEFAULT_TIMEOUT -from ._core_io import NDFileIO +# from ophyd_async.epics.adcore._core_logic import ADBaseDatasetDescriber +from ._core_io import ADBaseDatasetDescriber, NDFileIO, NDPluginBaseIO from ._utils import FileWriteMode -class ADWriter(DetectorWriter): +class ADWriterFormat(str, Enum): + HDF5 = ("HDF1:",) + TIFF = ("TIFF1:",) + + +NDFileIOT = TypeVar("NDFileIOT", bound=NDFileIO) +ADWriterT = TypeVar("ADWriterT", bound="ADWriter") + + +class ADWriter(DetectorWriter, Generic[NDFileIOT]): + default_suffix: str = "FILE1:" + def __init__( self, - fileio: NDFileIO, + fileio: NDFileIOT, path_provider: PathProvider, name_provider: NameProvider, dataset_describer: DatasetDescriber, file_extension: str = "", mimetype: str = "", + plugins: dict[str, NDPluginBaseIO] | None = None, ) -> None: - self.fileio = fileio + self._plugins = plugins + self._fileio = fileio self._path_provider = path_provider self._name_provider = name_provider self._dataset_describer = dataset_describer @@ -47,37 +63,58 @@ def __init__( self._multiplier = 1 self._filename_template = "%s%s_%6.6d" + @classmethod + def writer_and_io( + cls: type[ADWriterT], + prefix: str, + path_provider: PathProvider, + name_provider: NameProvider, + dataset_describer: ADBaseDatasetDescriber, + plugins: dict[str, NDPluginBaseIO] | None = None, + name: str = "", + ) -> tuple[ADWriterT, NDFileIOT]: + try: + fileio_cls = get_args(cls.__orig_bases__[0])[0] # type: ignore + except IndexError as err: + raise RuntimeError("File IO class for writer not specified!") from err + + fileio = fileio_cls(prefix, name=name) + writer = cls( + fileio, path_provider, name_provider, dataset_describer, plugins=plugins + ) + return writer, fileio + async def begin_capture(self) -> None: info = self._path_provider(device_name=self._name_provider()) - await self.fileio.enable_callbacks.set(True) + await self._fileio.enable_callbacks.set(True) # Set the directory creation depth first, since dir creation callback happens # when directory path PV is processed. - await self.fileio.create_directory.set(info.create_dir_depth) + await self._fileio.create_directory.set(info.create_dir_depth) await asyncio.gather( # See https://github.com/bluesky/ophyd-async/issues/122 - self.fileio.file_path.set(str(info.directory_path)), - self.fileio.file_name.set(info.filename), - self.fileio.file_write_mode.set(FileWriteMode.stream), + self._fileio.file_path.set(str(info.directory_path)), + self._fileio.file_name.set(info.filename), + self._fileio.file_write_mode.set(FileWriteMode.stream), # For non-HDF file writers, use AD file templating mechanism # for generating multi-image datasets - self.fileio.file_template.set( + self._fileio.file_template.set( self._filename_template + self._file_extension ), - self.fileio.auto_increment.set(True), - self.fileio.file_number.set(0), + self._fileio.auto_increment.set(True), + self._fileio.file_number.set(0), ) assert ( - await self.fileio.file_path_exists.get_value() + await self._fileio.file_path_exists.get_value() ), f"File path {info.directory_path} for file plugin does not exist!" # Overwrite num_capture to go forever - await self.fileio.num_capture.set(0) + await self._fileio.num_capture.set(0) # Wait for it to start, stashing the status that tells us when it finishes - self._capture_status = await set_and_wait_for_value(self.fileio.capture, True) + self._capture_status = await set_and_wait_for_value(self._fileio.capture, True) async def open(self, multiplier: int = 1) -> dict[str, DataKey]: self._emitted_resource = None @@ -91,7 +128,7 @@ async def open(self, multiplier: int = 1) -> dict[str, DataKey]: describe = { self._name_provider(): DataKey( source=self._name_provider(), - shape=tuple(frame_shape), + shape=list(frame_shape), dtype="array", dtype_numpy=dtype_numpy, external="STREAM:", @@ -103,11 +140,11 @@ async def observe_indices_written( self, timeout=DEFAULT_TIMEOUT ) -> AsyncGenerator[int, None]: """Wait until a specific index is ready to be collected""" - async for num_captured in observe_value(self.fileio.num_captured, timeout): + async for num_captured in observe_value(self._fileio.num_captured, timeout): yield num_captured // self._multiplier async def get_indices_written(self) -> int: - num_captured = await self.fileio.num_captured.get_value() + num_captured = await self._fileio.num_captured.get_value() return num_captured // self._multiplier async def collect_stream_docs( @@ -115,8 +152,8 @@ async def collect_stream_docs( ) -> AsyncIterator[StreamAsset]: if indices_written: if not self._emitted_resource: - file_path = Path(await self.fileio.file_path.get_value()) - file_name = await self.fileio.file_name.get_value() + file_path = Path(await self._fileio.file_path.get_value()) + file_name = await self._fileio.file_name.get_value() file_template = file_name + "_{:06d}" + self._file_extension frame_shape = await self._dataset_describer.shape() @@ -164,8 +201,8 @@ async def collect_stream_docs( async def close(self): # Already done a caput callback in _capture_status, so can't do one here - await self.fileio.capture.set(False, wait=False) - await wait_for_value(self.fileio.capture, False, DEFAULT_TIMEOUT) + await self._fileio.capture.set(False, wait=False) + await wait_for_value(self._fileio.capture, False, DEFAULT_TIMEOUT) if self._capture_status: # We kicked off an open, so wait for it to return await self._capture_status diff --git a/src/ophyd_async/epics/adcore/_hdf_writer.py b/src/ophyd_async/epics/adcore/_hdf_writer.py index 2481db44e1..f1e40a58e0 100644 --- a/src/ophyd_async/epics/adcore/_hdf_writer.py +++ b/src/ophyd_async/epics/adcore/_hdf_writer.py @@ -1,7 +1,6 @@ import asyncio from collections.abc import AsyncIterator from pathlib import Path -from typing import cast from xml.etree import ElementTree as ET from bluesky.protocols import Hints, StreamAsset @@ -17,7 +16,7 @@ wait_for_value, ) -from ._core_io import NDArrayBaseIO, NDFileHDFIO +from ._core_io import NDFileHDFIO, NDPluginBaseIO from ._core_writer import ADWriter from ._utils import ( convert_param_dtype_to_np, @@ -25,47 +24,43 @@ ) -class ADHDFWriter(ADWriter): +class ADHDFWriter(ADWriter[NDFileHDFIO]): + default_suffix: str = "HDF1:" + def __init__( self, - hdf: NDFileHDFIO, + fileio: NDFileHDFIO, path_provider: PathProvider, name_provider: NameProvider, dataset_describer: DatasetDescriber, - *plugins: NDArrayBaseIO, + plugins: dict[str, NDPluginBaseIO] | None = None, ) -> None: super().__init__( - hdf, + fileio, path_provider, name_provider, dataset_describer, - ".h5", - "application/x-hdf5", + plugins=plugins, + file_extension=".h5", + mimetype="application/x-hdf5", ) - self.hdf = cast(NDFileHDFIO, self.fileio) - self._plugins = plugins self._datasets: list[HDFDataset] = [] self._file: HDFFile | None = None self._include_file_number = False - @property - def include_file_number(self): - """Boolean property to toggle AD file number suffix""" - return self._include_file_number - async def open(self, multiplier: int = 1) -> dict[str, DataKey]: self._file = None # Setting HDF writer specific signals # Make sure we are using chunk auto-sizing - await asyncio.gather(self.hdf.chunk_size_auto.set(True)) + await asyncio.gather(self._fileio.chunk_size_auto.set(True)) await asyncio.gather( - self.hdf.num_extra_dims.set(0), - self.hdf.lazy_open.set(True), - self.hdf.swmr_mode.set(True), - self.hdf.xml_file_name.set(""), + self._fileio.num_extra_dims.set(0), + self._fileio.lazy_open.set(True), + self._fileio.swmr_mode.set(True), + self._fileio.xml_file_name.set(""), ) # By default, don't add file number to filename @@ -83,7 +78,7 @@ async def open(self, multiplier: int = 1) -> dict[str, DataKey]: outer_shape = (multiplier,) if multiplier > 1 else () # Determine number of frames that will be saved per HDF chunk - frames_per_chunk = await self.hdf.num_frames_chunks.get_value() + frames_per_chunk = await self._fileio.num_frames_chunks.get_value() # Add the main data self._datasets = [ @@ -97,39 +92,40 @@ async def open(self, multiplier: int = 1) -> dict[str, DataKey]: ) ] # And all the scalar datasets - for plugin in self._plugins: - maybe_xml = await plugin.nd_attributes_file.get_value() - # This is the check that ADCore does to see if it is an XML string - # rather than a filename to parse - if "" in maybe_xml: - root = ET.fromstring(maybe_xml) - for child in root: - datakey = child.attrib["name"] - if child.attrib.get("type", "EPICS_PV") == "EPICS_PV": - np_datatype = convert_pv_dtype_to_np( - child.attrib.get("dbrtype", "DBR_NATIVE") - ) - else: - np_datatype = convert_param_dtype_to_np( - child.attrib.get("datatype", "INT") - ) - self._datasets.append( - HDFDataset( - datakey, - f"/entry/instrument/NDAttributes/{datakey}", - (), - np_datatype, - multiplier, - # NDAttributes appear to always be configured with - # this chunk size - chunk_shape=(16384,), + if self._plugins is not None: + for plugin in self._plugins.values(): + maybe_xml = await plugin.nd_attributes_file.get_value() + # This is the check that ADCore does to see if it is an XML string + # rather than a filename to parse + if "" in maybe_xml: + root = ET.fromstring(maybe_xml) + for child in root: + datakey = child.attrib["name"] + if child.attrib.get("type", "EPICS_PV") == "EPICS_PV": + np_datatype = convert_pv_dtype_to_np( + child.attrib.get("dbrtype", "DBR_NATIVE") + ) + else: + np_datatype = convert_param_dtype_to_np( + child.attrib.get("datatype", "INT") + ) + self._datasets.append( + HDFDataset( + datakey, + f"/entry/instrument/NDAttributes/{datakey}", + (), + np_datatype, + multiplier, + # NDAttributes appear to always be configured with + # this chunk size + chunk_shape=(16384,), + ) ) - ) describe = { ds.data_key: DataKey( - source=self.hdf.full_file_name.source, - shape=outer_shape + tuple(ds.shape), + source=self._fileio.full_file_name.source, + shape=list(outer_shape + tuple(ds.shape)), dtype="array" if ds.shape else "number", dtype_numpy=ds.dtype_numpy, # type: ignore external="STREAM:", @@ -142,10 +138,10 @@ async def collect_stream_docs( self, indices_written: int ) -> AsyncIterator[StreamAsset]: # TODO: fail if we get dropped frames - await self.hdf.flush_now.set(True) + await self._fileio.flush_now.set(True) if indices_written: if not self._file: - path = Path(await self.hdf.full_file_name.get_value()) + path = Path(await self._fileio.full_file_name.get_value()) self._file = HDFFile( # See https://github.com/bluesky/ophyd-async/issues/122 path, @@ -162,8 +158,8 @@ async def collect_stream_docs( async def close(self): # Already done a caput callback in _capture_status, so can't do one here - await self.hdf.capture.set(False, wait=False) - await wait_for_value(self.hdf.capture, False, DEFAULT_TIMEOUT) + await self._fileio.capture.set(False, wait=False) + await wait_for_value(self._fileio.capture, False, DEFAULT_TIMEOUT) if self._capture_status: # We kicked off an open, so wait for it to return await self._capture_status diff --git a/src/ophyd_async/epics/adcore/_single_trigger.py b/src/ophyd_async/epics/adcore/_single_trigger.py index c39e4e46ab..72ef80eca8 100644 --- a/src/ophyd_async/epics/adcore/_single_trigger.py +++ b/src/ophyd_async/epics/adcore/_single_trigger.py @@ -21,10 +21,13 @@ def __init__( drv: ADBaseIO, read_uncached: Sequence[SignalR] = (), name="", - **plugins: NDPluginBaseIO, + plugins: dict[str, NDPluginBaseIO] | None = None, ) -> None: self.drv = drv - self.__dict__.update(plugins) + + if plugins is not None: + for name, plugin in plugins.items(): + setattr(self, name, plugin) self.add_readables( [self.drv.array_counter, *read_uncached], diff --git a/src/ophyd_async/epics/adcore/_tiff_writer.py b/src/ophyd_async/epics/adcore/_tiff_writer.py index 5dd6cb9225..8ccb681c95 100644 --- a/src/ophyd_async/epics/adcore/_tiff_writer.py +++ b/src/ophyd_async/epics/adcore/_tiff_writer.py @@ -1,23 +1,27 @@ from ophyd_async.core import DatasetDescriber, NameProvider, PathProvider -from ._core_io import NDFileIO +from ._core_io import NDFileIO, NDPluginBaseIO from ._core_writer import ADWriter -class ADTIFFWriter(ADWriter): +class ADTIFFWriter(ADWriter[NDFileIO]): + default_suffix: str = "TIFF1:" + def __init__( self, - fileio: NDFileIO, + prefix, path_provider: PathProvider, name_provider: NameProvider, dataset_describer: DatasetDescriber, + plugins: dict[str, NDPluginBaseIO] | None = None, ) -> None: super().__init__( - fileio, + prefix, path_provider, name_provider, dataset_describer, - ".tiff", - "multipart/related;type=image/tiff", + plugins=plugins, + file_extension=".tiff", + mimetype="multipart/related;type=image/tiff", ) - self.tiff = self.fileio + self.tiff = self._fileio From 38a61e861c5e15905eea03e535fb0fca34ea7212 Mon Sep 17 00:00:00 2001 From: Jakub Wlodek Date: Fri, 22 Nov 2024 13:52:50 -0500 Subject: [PATCH 14/60] Update detector and controller classes to reflect changes --- src/ophyd_async/epics/adaravis/_aravis.py | 86 ++++++------------- .../epics/adaravis/_aravis_controller.py | 55 +++++++++--- src/ophyd_async/epics/adkinetix/__init__.py | 3 +- src/ophyd_async/epics/adkinetix/_kinetix.py | 58 ++++--------- .../epics/adkinetix/_kinetix_controller.py | 20 ++--- src/ophyd_async/epics/adpilatus/__init__.py | 4 +- src/ophyd_async/epics/adpilatus/_pilatus.py | 76 +++++----------- .../epics/adpilatus/_pilatus_controller.py | 52 ++++++++--- .../epics/adsimdetector/__init__.py | 5 +- src/ophyd_async/epics/adsimdetector/_sim.py | 64 +++++++------- src/ophyd_async/epics/advimba/_vimba.py | 60 +++++-------- .../epics/advimba/_vimba_controller.py | 26 +++--- 12 files changed, 223 insertions(+), 286 deletions(-) diff --git a/src/ophyd_async/epics/adaravis/_aravis.py b/src/ophyd_async/epics/adaravis/_aravis.py index d72a2f3505..923dda2d59 100644 --- a/src/ophyd_async/epics/adaravis/_aravis.py +++ b/src/ophyd_async/epics/adaravis/_aravis.py @@ -1,15 +1,18 @@ from collections.abc import Sequence -from typing import cast, get_args from ophyd_async.core import PathProvider from ophyd_async.core._signal import SignalR -from ophyd_async.epics import adcore +from ophyd_async.epics.adcore._core_detector import AreaDetector +from ophyd_async.epics.adcore._core_io import NDPluginBaseIO + +# from ophyd_async.epics.adcore._core_logic import ad_driver_factory, ad_writer_factory +from ophyd_async.epics.adcore._core_writer import ADWriter +from ophyd_async.epics.adcore._hdf_writer import ADHDFWriter from ._aravis_controller import AravisController -from ._aravis_io import AravisDriverIO -class AravisDetector(adcore.AreaDetector): +class AravisDetector(AreaDetector[AravisController, ADWriter]): """ Ophyd-async implementation of an ADAravis Detector. The detector may be configured for an external trigger on a GPIO port, @@ -21,70 +24,29 @@ def __init__( prefix: str, path_provider: PathProvider, drv_suffix="cam1:", - hdf_suffix="HDF1:", - name="", + writer_cls: type[ADWriter] = ADHDFWriter, + fileio_suffix: str | None = None, + name: str = "", gpio_number: AravisController.GPIO_NUMBER = 1, config_sigs: Sequence[SignalR] = (), + plugins: dict[str, NDPluginBaseIO] = None, ): - super().__init__( - prefix, - path_provider, - adcore.ADHDFWriter, - hdf_suffix, - AravisController, - AravisDriverIO, - drv_suffix=drv_suffix, - name=name, - config_sigs=config_sigs, - gpio_number=gpio_number, + if plugins is None: + plugins = {} + controller, driver = AravisController.controller_and_drv( + prefix + drv_suffix, gpio_number=gpio_number, name=name ) - self.hdf = self._fileio - - @property - def controller(self) -> AravisController: - return cast(AravisController, self._controller) - - def get_external_trigger_gpio(self): - return self.controller.gpio_number - def set_external_trigger_gpio(self, gpio_number: AravisController.GPIO_NUMBER): - supported_gpio_numbers = get_args(AravisController.GPIO_NUMBER) - if gpio_number not in supported_gpio_numbers: - raise ValueError( - f"{self.__class__.__name__} only supports the following GPIO " - f"indices: {supported_gpio_numbers} but was asked to " - f"use {gpio_number}" - ) - self.controller.gpio_number = gpio_number - - -class AravisDetectorTIFF(adcore.AreaDetector): - """ - Ophyd-async implementation of an ADAravis Detector. - The detector may be configured for an external trigger on a GPIO port, - which must be done prior to preparing the detector - """ - - def __init__( - self, - prefix: str, - path_provider: PathProvider, - drv_suffix="cam1:", - hdf_suffix="HDF1:", - name="", - gpio_number: AravisController.GPIO_NUMBER = 1, - config_sigs: Sequence[SignalR] = (), - ): super().__init__( - prefix, - path_provider, - adcore.ADTIFFWriter, - hdf_suffix, - AravisController, - AravisDriverIO, - drv_suffix=drv_suffix, + prefix=prefix, + driver=driver, + controller=controller, + writer_cls=writer_cls, + fileio_suffix=fileio_suffix, + path_provider=path_provider, + plugins=plugins, name=name, config_sigs=config_sigs, - gpio_number=gpio_number, ) - self.tiff = self._fileio + + self.drv = driver diff --git a/src/ophyd_async/epics/adaravis/_aravis_controller.py b/src/ophyd_async/epics/adaravis/_aravis_controller.py index d6f3de2963..2f0b952949 100644 --- a/src/ophyd_async/epics/adaravis/_aravis_controller.py +++ b/src/ophyd_async/epics/adaravis/_aravis_controller.py @@ -1,11 +1,13 @@ import asyncio -from typing import Literal, cast +from typing import Literal, TypeVar, get_args from ophyd_async.core import ( DetectorTrigger, TriggerInfo, ) from ophyd_async.epics import adcore +from ophyd_async.epics.adcore._core_io import DetectorState +from ophyd_async.epics.adcore._core_logic import DEFAULT_GOOD_STATES from ._aravis_io import AravisDriverIO, AravisTriggerMode, AravisTriggerSource @@ -14,17 +16,33 @@ # runtime. See https://github.com/bluesky/ophyd-async/issues/308 _HIGHEST_POSSIBLE_DEADTIME = 1961e-6 +AravisControllerT = TypeVar("AravisControllerT", bound="AravisController") -class AravisController(adcore.ADBaseController): + +class AravisController(adcore.ADBaseController[AravisDriverIO]): GPIO_NUMBER = Literal[1, 2, 3, 4] - def __init__(self, driver: AravisDriverIO, gpio_number: GPIO_NUMBER) -> None: - super().__init__(driver) + def __init__( + self, + driver: AravisDriverIO, + good_states: frozenset[DetectorState] = DEFAULT_GOOD_STATES, + gpio_number: GPIO_NUMBER = 1, + ) -> None: + super().__init__(driver, good_states=good_states) self.gpio_number = gpio_number - @property - def driver(self) -> AravisDriverIO: - return cast(AravisDriverIO, self._driver) + @classmethod + def controller_and_drv( + cls: type[AravisControllerT], + prefix: str, + good_states: frozenset[DetectorState] = DEFAULT_GOOD_STATES, + name: str = "", + gpio_number: GPIO_NUMBER = 1, + ) -> tuple[AravisControllerT, AravisDriverIO]: + driver_cls = get_args(cls.__orig_bases__[0])[0] # type: ignore + driver = driver_cls(prefix, name=name) + controller = cls(driver, good_states=good_states, gpio_number=gpio_number) + return controller, driver def get_deadtime(self, exposure: float | None) -> float: return _HIGHEST_POSSIBLE_DEADTIME @@ -35,16 +53,16 @@ async def prepare(self, trigger_info: TriggerInfo): else: image_mode = adcore.ImageMode.multiple if (exposure := trigger_info.livetime) is not None: - await self.driver.acquire_time.set(exposure) + await self._driver.acquire_time.set(exposure) trigger_mode, trigger_source = self._get_trigger_info(trigger_info.trigger) # trigger mode must be set first and on it's own! - await self.driver.trigger_mode.set(trigger_mode) + await self._driver.trigger_mode.set(trigger_mode) await asyncio.gather( - self.driver.trigger_source.set(trigger_source), - self.driver.num_images.set(trigger_info.total_number_of_triggers), - self.driver.image_mode.set(image_mode), + self._driver.trigger_source.set(trigger_source), + self._driver.num_images.set(trigger_info.total_number_of_triggers), + self._driver.image_mode.set(image_mode), ) def _get_trigger_info( @@ -65,3 +83,16 @@ def _get_trigger_info( return AravisTriggerMode.off, "Freerun" else: return (AravisTriggerMode.on, f"Line{self.gpio_number}") # type: ignore + + def get_external_trigger_gpio(self): + return self.gpio_number + + def set_external_trigger_gpio(self, gpio_number: GPIO_NUMBER): + supported_gpio_numbers = get_args(AravisController.GPIO_NUMBER) + if gpio_number not in supported_gpio_numbers: + raise ValueError( + f"{self.__class__.__name__} only supports the following GPIO " + f"indices: {supported_gpio_numbers} but was asked to " + f"use {gpio_number}" + ) + self.gpio_number = gpio_number diff --git a/src/ophyd_async/epics/adkinetix/__init__.py b/src/ophyd_async/epics/adkinetix/__init__.py index e69dc95136..7747be3356 100644 --- a/src/ophyd_async/epics/adkinetix/__init__.py +++ b/src/ophyd_async/epics/adkinetix/__init__.py @@ -1,10 +1,9 @@ -from ._kinetix import KinetixDetector, KinetixDetectorTIFF +from ._kinetix import KinetixDetector from ._kinetix_controller import KinetixController from ._kinetix_io import KinetixDriverIO __all__ = [ "KinetixDetector", - "KinetixDetectorTIFF", "KinetixController", "KinetixDriverIO", ] diff --git a/src/ophyd_async/epics/adkinetix/_kinetix.py b/src/ophyd_async/epics/adkinetix/_kinetix.py index b53f43582b..b2823853a5 100644 --- a/src/ophyd_async/epics/adkinetix/_kinetix.py +++ b/src/ophyd_async/epics/adkinetix/_kinetix.py @@ -1,13 +1,12 @@ from collections.abc import Sequence from ophyd_async.core import PathProvider, SignalR -from ophyd_async.epics import adcore +from ophyd_async.epics.adcore import ADHDFWriter, ADWriter, AreaDetector, NDPluginBaseIO from ._kinetix_controller import KinetixController -from ._kinetix_io import KinetixDriverIO -class KinetixDetector(adcore.AreaDetector): +class KinetixDetector(AreaDetector[KinetixController, ADWriter]): """ Ophyd-async implementation of an ADKinetix Detector. https://github.com/NSLS-II/ADKinetix @@ -17,49 +16,26 @@ def __init__( self, prefix: str, path_provider: PathProvider, - drv_suffix="cam1:", - hdf_suffix="HDF1:", - name="", + drv_suffix: str = "cam1:", + writer_cls: type[ADWriter] = ADHDFWriter, + fileio_suffix: str | None = None, + name: str = "", + plugins: dict[str, NDPluginBaseIO] | None = None, config_sigs: Sequence[SignalR] = (), ): - super().__init__( - prefix, - path_provider, - adcore.ADHDFWriter, - hdf_suffix, - KinetixController, - KinetixDriverIO, - drv_suffix=drv_suffix, - name=name, - config_sigs=config_sigs, + controller, driver = KinetixController.controller_and_drv( + prefix + drv_suffix, name=name ) - self.hdf = self._fileio - -class KinetixDetectorTIFF(adcore.AreaDetector): - """ - Ophyd-async implementation of an ADKinetix Detector. - https://github.com/NSLS-II/ADKinetix - """ - - def __init__( - self, - prefix: str, - path_provider: PathProvider, - drv_suffix="cam1:", - tiff_suffix="TIFF1:", - name="", - config_sigs: Sequence[SignalR] = (), - ): super().__init__( - prefix, - path_provider, - adcore.ADTIFFWriter, - tiff_suffix, - KinetixController, - KinetixDriverIO, - drv_suffix=drv_suffix, + prefix=prefix, + driver=driver, + controller=controller, + writer_cls=writer_cls, + path_provider=path_provider, + plugins=plugins, name=name, + fileio_suffix=fileio_suffix, config_sigs=config_sigs, ) - self.tiff = self._fileio + self.drv = driver diff --git a/src/ophyd_async/epics/adkinetix/_kinetix_controller.py b/src/ophyd_async/epics/adkinetix/_kinetix_controller.py index 52c24ad9c3..8243c6bb4d 100644 --- a/src/ophyd_async/epics/adkinetix/_kinetix_controller.py +++ b/src/ophyd_async/epics/adkinetix/_kinetix_controller.py @@ -1,9 +1,10 @@ import asyncio -from typing import cast from ophyd_async.core import DetectorTrigger from ophyd_async.core._detector import TriggerInfo from ophyd_async.epics import adcore +from ophyd_async.epics.adcore._core_io import DetectorState +from ophyd_async.epics.adcore._core_logic import DEFAULT_GOOD_STATES from ._kinetix_io import KinetixDriverIO, KinetixTriggerMode @@ -15,30 +16,27 @@ } -class KinetixController(adcore.ADBaseController): +class KinetixController(adcore.ADBaseController[KinetixDriverIO]): def __init__( self, driver: KinetixDriverIO, + good_states: frozenset[DetectorState] = DEFAULT_GOOD_STATES, ) -> None: - super().__init__(driver) - - @property - def driver(self) -> KinetixDriverIO: - return cast(KinetixDriverIO, self._driver) + super().__init__(driver, good_states=good_states) def get_deadtime(self, exposure: float | None) -> float: return 0.001 async def prepare(self, trigger_info: TriggerInfo): await asyncio.gather( - self.driver.trigger_mode.set( + self._driver.trigger_mode.set( KINETIX_TRIGGER_MODE_MAP[trigger_info.trigger] ), - self.driver.num_images.set(trigger_info.total_number_of_triggers), - self.driver.image_mode.set(adcore.ImageMode.multiple), + self._driver.num_images.set(trigger_info.total_number_of_triggers), + self._driver.image_mode.set(adcore.ImageMode.multiple), ) if trigger_info.livetime is not None and trigger_info.trigger not in [ DetectorTrigger.variable_gate, DetectorTrigger.constant_gate, ]: - await self.driver.acquire_time.set(trigger_info.livetime) + await self._driver.acquire_time.set(trigger_info.livetime) diff --git a/src/ophyd_async/epics/adpilatus/__init__.py b/src/ophyd_async/epics/adpilatus/__init__.py index 66bcd2feff..966bcf37b7 100644 --- a/src/ophyd_async/epics/adpilatus/__init__.py +++ b/src/ophyd_async/epics/adpilatus/__init__.py @@ -1,5 +1,5 @@ -from ._pilatus import PilatusDetector, PilatusReadoutTime -from ._pilatus_controller import PilatusController +from ._pilatus import PilatusDetector +from ._pilatus_controller import PilatusController, PilatusReadoutTime from ._pilatus_io import PilatusDriverIO, PilatusTriggerMode __all__ = [ diff --git a/src/ophyd_async/epics/adpilatus/_pilatus.py b/src/ophyd_async/epics/adpilatus/_pilatus.py index 86c6b44f31..50435ed572 100644 --- a/src/ophyd_async/epics/adpilatus/_pilatus.py +++ b/src/ophyd_async/epics/adpilatus/_pilatus.py @@ -1,79 +1,45 @@ from collections.abc import Sequence -from enum import Enum from ophyd_async.core import PathProvider from ophyd_async.core._signal import SignalR -from ophyd_async.epics import adcore +from ophyd_async.epics.adcore._core_detector import AreaDetector +from ophyd_async.epics.adcore._core_io import NDPluginBaseIO +from ophyd_async.epics.adcore._core_writer import ADWriter +from ophyd_async.epics.adcore._hdf_writer import ADHDFWriter from ._pilatus_controller import PilatusController -from ._pilatus_io import PilatusDriverIO -#: Cite: https://media.dectris.com/User_Manual-PILATUS2-V1_4.pdf -#: The required minimum time difference between ExpPeriod and ExpTime -#: (readout time) is 2.28 ms -#: We provide an option to override for newer Pilatus models -class PilatusReadoutTime(float, Enum): - """Pilatus readout time per model in ms""" - - # Cite: https://media.dectris.com/User_Manual-PILATUS2-V1_4.pdf - pilatus2 = 2.28e-3 - - # Cite: https://media.dectris.com/user-manual-pilatus3-2020.pdf - pilatus3 = 0.95e-3 - - -class PilatusDetector(adcore.AreaDetector): +class PilatusDetector(AreaDetector[PilatusController, ADWriter]): """A Pilatus StandardDetector writing HDF files""" def __init__( self, prefix: str, path_provider: PathProvider, + readout_time: float, drv_suffix: str = "cam1:", - hdf_suffix: str = "HDF1:", + writer_cls: type[ADWriter] = ADHDFWriter, + fileio_suffix: str | None = None, name: str = "", + plugins: dict[str, NDPluginBaseIO] = None, config_sigs: Sequence[SignalR] = (), - readout_time: PilatusReadoutTime = PilatusReadoutTime.pilatus3, ): - super().__init__( - prefix, - path_provider, - adcore.ADHDFWriter, - hdf_suffix, - PilatusController, - PilatusDriverIO, - drv_suffix=drv_suffix, - name=name, - config_sigs=config_sigs, - readout_time=readout_time, + if plugins is None: + plugins = {} + controller, driver = PilatusController.controller_and_drv( + prefix + drv_suffix, name=name, readout_time=readout_time ) - self.hdf = self._fileio - - -class PilatusDetectorTIFF(adcore.AreaDetector): - """A Pilatus StandardDetector writing HDF files""" - def __init__( - self, - prefix: str, - path_provider: PathProvider, - drv_suffix: str = "cam1:", - tiff_suffix: str = "TIFF1:", - name: str = "", - config_sigs: Sequence[SignalR] = (), - readout_time: PilatusReadoutTime = PilatusReadoutTime.pilatus3, - ): super().__init__( - prefix, - path_provider, - adcore.ADTIFFWriter, - tiff_suffix, - PilatusController, - PilatusDriverIO, - drv_suffix=drv_suffix, + prefix=prefix, + driver=driver, + controller=controller, + writer_cls=writer_cls, + path_provider=path_provider, + plugins=plugins, name=name, + fileio_suffix=fileio_suffix, config_sigs=config_sigs, - readout_time=readout_time, ) - self.tiff = self._fileio + self.drv = driver diff --git a/src/ophyd_async/epics/adpilatus/_pilatus_controller.py b/src/ophyd_async/epics/adpilatus/_pilatus_controller.py index ea43a36a75..8eaa4ddd5a 100644 --- a/src/ophyd_async/epics/adpilatus/_pilatus_controller.py +++ b/src/ophyd_async/epics/adpilatus/_pilatus_controller.py @@ -1,5 +1,6 @@ import asyncio -from typing import cast +from enum import Enum +from typing import TypeVar, get_args from ophyd_async.core import ( DEFAULT_TIMEOUT, @@ -8,11 +9,30 @@ ) from ophyd_async.core._detector import TriggerInfo from ophyd_async.epics import adcore +from ophyd_async.epics.adcore._core_io import DetectorState +from ophyd_async.epics.adcore._core_logic import DEFAULT_GOOD_STATES from ._pilatus_io import PilatusDriverIO, PilatusTriggerMode -class PilatusController(adcore.ADBaseController): +#: Cite: https://media.dectris.com/User_Manual-PILATUS2-V1_4.pdf +#: The required minimum time difference between ExpPeriod and ExpTime +#: (readout time) is 2.28 ms +#: We provide an option to override for newer Pilatus models +class PilatusReadoutTime(float, Enum): + """Pilatus readout time per model in ms""" + + # Cite: https://media.dectris.com/User_Manual-PILATUS2-V1_4.pdf + pilatus2 = 2.28e-3 + + # Cite: https://media.dectris.com/user-manual-pilatus3-2020.pdf + pilatus3 = 0.95e-3 + + +PilatusControllerT = TypeVar("PilatusControllerT", bound="PilatusController") + + +class PilatusController(adcore.ADBaseController[PilatusDriverIO]): _supported_trigger_types = { DetectorTrigger.internal: PilatusTriggerMode.internal, DetectorTrigger.constant_gate: PilatusTriggerMode.ext_enable, @@ -22,14 +42,24 @@ class PilatusController(adcore.ADBaseController): def __init__( self, driver: PilatusDriverIO, - readout_time: float, + good_states: frozenset[DetectorState] = DEFAULT_GOOD_STATES, + readout_time: float = PilatusReadoutTime.pilatus3, ) -> None: - super().__init__(driver) + super().__init__(driver, good_states=good_states) self._readout_time = readout_time - @property - def driver(self) -> PilatusDriverIO: - return cast(PilatusDriverIO, self._driver) + @classmethod + def controller_and_drv( + cls: type[PilatusControllerT], + prefix: str, + good_states: frozenset[DetectorState] = DEFAULT_GOOD_STATES, + name: str = "", + readout_time: float = PilatusReadoutTime.pilatus3, + ) -> tuple[PilatusControllerT, PilatusDriverIO]: + driver_cls = get_args(cls.__orig_bases__[0])[0] # type: ignore + driver = driver_cls(prefix, name=name) + controller = cls(driver, good_states=good_states, readout_time=readout_time) + return controller, driver def get_deadtime(self, exposure: float | None) -> float: return self._readout_time @@ -40,13 +70,13 @@ async def prepare(self, trigger_info: TriggerInfo): trigger_info.livetime ) await asyncio.gather( - self.driver.trigger_mode.set(self._get_trigger_mode(trigger_info.trigger)), - self.driver.num_images.set( + self._driver.trigger_mode.set(self._get_trigger_mode(trigger_info.trigger)), + self._driver.num_images.set( 999_999 if trigger_info.total_number_of_triggers == 0 else trigger_info.total_number_of_triggers ), - self.driver.image_mode.set(adcore.ImageMode.multiple), + self._driver.image_mode.set(adcore.ImageMode.multiple), ) async def arm(self): @@ -57,7 +87,7 @@ async def arm(self): # is actually ready. Should wait for that too or we risk dropping # a frame await wait_for_value( - self.driver.armed, + self._driver.armed, True, timeout=DEFAULT_TIMEOUT, ) diff --git a/src/ophyd_async/epics/adsimdetector/__init__.py b/src/ophyd_async/epics/adsimdetector/__init__.py index ae242ffddf..0d75133fea 100644 --- a/src/ophyd_async/epics/adsimdetector/__init__.py +++ b/src/ophyd_async/epics/adsimdetector/__init__.py @@ -1,6 +1,7 @@ -from ._sim import SimDetector, SimDetectorTIFF +from ._sim import SimController, SimDetector, SimDriverIO __all__ = [ + "SimDriverIO", + "SimController", "SimDetector", - "SimDetectorTIFF", ] diff --git a/src/ophyd_async/epics/adsimdetector/_sim.py b/src/ophyd_async/epics/adsimdetector/_sim.py index 56351a8c56..bb584347f7 100644 --- a/src/ophyd_async/epics/adsimdetector/_sim.py +++ b/src/ophyd_async/epics/adsimdetector/_sim.py @@ -2,55 +2,51 @@ from ophyd_async.core import PathProvider, SignalR from ophyd_async.epics import adcore +from ophyd_async.epics.adcore._core_detector import AreaDetector +from ophyd_async.epics.adcore._core_io import DetectorState, NDPluginBaseIO +from ophyd_async.epics.adcore._core_logic import DEFAULT_GOOD_STATES +from ophyd_async.epics.adcore._core_writer import ADWriter +from ophyd_async.epics.adcore._hdf_writer import ADHDFWriter -class SimDetector(adcore.AreaDetector): +class SimDriverIO(adcore.ADBaseIO): ... + +class SimController(adcore.ADBaseController[SimDriverIO]): def __init__( self, - prefix: str, - path_provider: PathProvider, - hdf_suffix:str="HDF1:", - drv_suffix:str="cam1:", - name: str = "", - config_sigs: Sequence[SignalR] = (), - ): - - super().__init__( - prefix, - path_provider, - adcore.ADHDFWriter, - hdf_suffix, - adcore.ADBaseController, - adcore.ADBaseIO, - drv_suffix=drv_suffix, - name=name, - config_sigs=config_sigs, - ) - self.hdf = self._fileio - + driver: SimDriverIO, + good_states: frozenset[DetectorState] = DEFAULT_GOOD_STATES, + ) -> None: + super().__init__(driver, good_states=good_states) -class SimDetectorTIFF(adcore.AreaDetector): +class SimDetector(AreaDetector[SimController, ADWriter]): def __init__( self, prefix: str, path_provider: PathProvider, - tiff_suffix:str="TIFF1:", - drv_suffix:str="cam1:", - name: str = "", + drv_suffix="cam1:", + writer_cls: type[ADWriter] = ADHDFWriter, + fileio_suffix: str | None = None, + name="", config_sigs: Sequence[SignalR] = (), + plugins: dict[str, NDPluginBaseIO] = None, ): + if plugins is None: + plugins = {} + controller, driver = SimController.controller_and_drv( + prefix + drv_suffix, name=name + ) super().__init__( - prefix, - path_provider, - adcore.ADTIFFWriter, - tiff_suffix, - adcore.ADBaseController, - adcore.ADBaseIO, - drv_suffix=drv_suffix, + prefix=prefix, + driver=driver, + controller=controller, + writer_cls=writer_cls, + fileio_suffix=fileio_suffix, + path_provider=path_provider, + plugins=plugins, name=name, config_sigs=config_sigs, ) - self.tiff = self._fileio diff --git a/src/ophyd_async/epics/advimba/_vimba.py b/src/ophyd_async/epics/advimba/_vimba.py index 4f48bcaefd..ea9a86c635 100644 --- a/src/ophyd_async/epics/advimba/_vimba.py +++ b/src/ophyd_async/epics/advimba/_vimba.py @@ -1,13 +1,12 @@ from collections.abc import Sequence from ophyd_async.core import PathProvider, SignalR -from ophyd_async.epics import adcore +from ophyd_async.epics.adcore import ADHDFWriter, ADWriter, AreaDetector, NDPluginBaseIO from ._vimba_controller import VimbaController -from ._vimba_io import VimbaDriverIO -class VimbaDetector(adcore.AreaDetector): +class VimbaDetector(AreaDetector[VimbaController, ADWriter]): """ Ophyd-async implementation of an ADVimba Detector. """ @@ -16,48 +15,29 @@ def __init__( self, prefix: str, path_provider: PathProvider, - drv_suffix="cam1:", - hdf_suffix="HDF1:", - name="", + drv_suffix: str = "cam1:", + writer_cls: type[ADWriter] = ADHDFWriter, + fileio_suffix: str | None = None, + name: str = "", + plugins: dict[str, NDPluginBaseIO] = None, config_sigs: Sequence[SignalR] = (), ): - super().__init__( - prefix, - path_provider, - adcore.ADHDFWriter, - hdf_suffix, - VimbaController, - VimbaDriverIO, - drv_suffix=drv_suffix, - name=name, - config_sigs=config_sigs, + if plugins is None: + plugins = {} + controller, driver = VimbaController.controller_and_drv( + prefix + drv_suffix, name=name ) - self.hdf = self._fileio - - -class VimbaDetectorTIFF(adcore.AreaDetector): - """ - Ophyd-async implementation of an ADVimba Detector. - """ - def __init__( - self, - prefix: str, - path_provider: PathProvider, - drv_suffix="cam1:", - tiff_suffix="TIFF1:", - name="", - config_sigs: Sequence[SignalR] = (), - ): super().__init__( - prefix, - path_provider, - adcore.ADTIFFWriter, - tiff_suffix, - VimbaController, - VimbaDriverIO, - drv_suffix=drv_suffix, + prefix=prefix, + driver=driver, + controller=controller, + writer_cls=writer_cls, + path_provider=path_provider, + plugins=plugins, name=name, + fileio_suffix=fileio_suffix, config_sigs=config_sigs, ) - self.tiff = self._fileio + + self.drv = driver diff --git a/src/ophyd_async/epics/advimba/_vimba_controller.py b/src/ophyd_async/epics/advimba/_vimba_controller.py index 882def31ff..5a65205888 100644 --- a/src/ophyd_async/epics/advimba/_vimba_controller.py +++ b/src/ophyd_async/epics/advimba/_vimba_controller.py @@ -1,9 +1,10 @@ import asyncio -from typing import cast from ophyd_async.core import DetectorTrigger from ophyd_async.core._detector import TriggerInfo from ophyd_async.epics import adcore +from ophyd_async.epics.adcore._core_io import DetectorState +from ophyd_async.epics.adcore._core_logic import DEFAULT_GOOD_STATES from ._vimba_io import VimbaDriverIO, VimbaExposeOutMode, VimbaOnOff, VimbaTriggerSource @@ -22,33 +23,30 @@ } -class VimbaController(adcore.ADBaseController): +class VimbaController(adcore.ADBaseController[VimbaDriverIO]): def __init__( self, driver: VimbaDriverIO, + good_states: frozenset[DetectorState] = DEFAULT_GOOD_STATES, ) -> None: - super().__init__(driver) - - @property - def driver(self) -> VimbaDriverIO: - return cast(VimbaDriverIO, self._driver) + super().__init__(driver, good_states=good_states) def get_deadtime(self, exposure: float | None) -> float: return 0.001 async def prepare(self, trigger_info: TriggerInfo): await asyncio.gather( - self.driver.trigger_mode.set(TRIGGER_MODE[trigger_info.trigger]), - self.driver.exposure_mode.set(EXPOSE_OUT_MODE[trigger_info.trigger]), - self.driver.num_images.set(trigger_info.total_number_of_triggers), - self.driver.image_mode.set(adcore.ImageMode.multiple), + self._driver.trigger_mode.set(TRIGGER_MODE[trigger_info.trigger]), + self._driver.exposure_mode.set(EXPOSE_OUT_MODE[trigger_info.trigger]), + self._driver.num_images.set(trigger_info.total_number_of_triggers), + self._driver.image_mode.set(adcore.ImageMode.multiple), ) if trigger_info.livetime is not None and trigger_info.trigger not in [ DetectorTrigger.variable_gate, DetectorTrigger.constant_gate, ]: - await self.driver.acquire_time.set(trigger_info.livetime) + await self._driver.acquire_time.set(trigger_info.livetime) if trigger_info.trigger != DetectorTrigger.internal: - self.driver.trigger_source.set(VimbaTriggerSource.line1) + self._driver.trigger_source.set(VimbaTriggerSource.line1) else: - self.driver.trigger_source.set(VimbaTriggerSource.freerun) + self._driver.trigger_source.set(VimbaTriggerSource.freerun) From aecdf0491bd2da0e03a11d92ba7e5be868e88553 Mon Sep 17 00:00:00 2001 From: Jakub Wlodek Date: Fri, 22 Nov 2024 13:53:33 -0500 Subject: [PATCH 15/60] Make sure panda standard det uses new type hints --- src/ophyd_async/fastcs/panda/_hdf_panda.py | 4 +++- src/ophyd_async/plan_stubs/_nd_attributes.py | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/ophyd_async/fastcs/panda/_hdf_panda.py b/src/ophyd_async/fastcs/panda/_hdf_panda.py index 5045d7b27f..419d4c9890 100644 --- a/src/ophyd_async/fastcs/panda/_hdf_panda.py +++ b/src/ophyd_async/fastcs/panda/_hdf_panda.py @@ -10,7 +10,9 @@ from ._writer import PandaHDFWriter -class HDFPanda(CommonPandaBlocks, StandardDetector): +class HDFPanda( + CommonPandaBlocks, StandardDetector[PandaPcapController, PandaHDFWriter] +): def __init__( self, prefix: str, diff --git a/src/ophyd_async/plan_stubs/_nd_attributes.py b/src/ophyd_async/plan_stubs/_nd_attributes.py index 95a473033d..6aa8ff2027 100644 --- a/src/ophyd_async/plan_stubs/_nd_attributes.py +++ b/src/ophyd_async/plan_stubs/_nd_attributes.py @@ -49,9 +49,9 @@ def setup_ndattributes( def setup_ndstats_sum(detector: Device): - hdf = getattr(detector, "hdf", None) + hdf = getattr(detector, "fileio", None) assert isinstance(hdf, NDFileHDFIO), ( - f"Expected {detector.name} to have 'hdf' attribute that is an NDFilHDFIO, " + f"Expected {detector.name} to have 'fileio' attribute that is an NDFilHDFIO, " f"got {hdf}" ) yield from ( From e42fa128941efc5bb83dc64b11dde488fe78c0ce Mon Sep 17 00:00:00 2001 From: Jakub Wlodek Date: Fri, 22 Nov 2024 14:32:14 -0500 Subject: [PATCH 16/60] Most tests passing --- src/ophyd_async/plan_stubs/_fly.py | 2 +- tests/core/test_flyer.py | 14 ++-- tests/epics/adaravis/test_aravis.py | 24 +++--- tests/epics/adcore/test_drivers.py | 10 +-- tests/epics/adcore/test_scans.py | 13 ++-- tests/epics/adcore/test_single_trigger.py | 22 +++--- tests/epics/adcore/test_writers.py | 34 +++++---- tests/epics/adkinetix/test_kinetix.py | 14 ++-- tests/epics/adpilatus/test_pilatus.py | 26 +++---- tests/epics/adsimdetector/test_sim.py | 90 ++++++++++++----------- tests/epics/advimba/test_vimba.py | 14 ++-- tests/epics/conftest.py | 58 +++++++++------ tests/epics/eiger/test_eiger_detector.py | 2 +- tests/fastcs/panda/test_hdf_panda.py | 6 +- tests/plan_stubs/test_fly.py | 14 ++-- tests/sim/test_sim_detector.py | 4 +- tests/sim/test_streaming_plan.py | 2 +- 17 files changed, 189 insertions(+), 160 deletions(-) diff --git a/src/ophyd_async/plan_stubs/_fly.py b/src/ophyd_async/plan_stubs/_fly.py index 043da476f4..41ccf115b8 100644 --- a/src/ophyd_async/plan_stubs/_fly.py +++ b/src/ophyd_async/plan_stubs/_fly.py @@ -58,7 +58,7 @@ def prepare_static_seq_table_flyer_and_detectors_with_same_trigger( if not detectors: raise ValueError("No detectors provided. There must be at least one.") - deadtime = max(det.controller.get_deadtime(exposure) for det in detectors) + deadtime = max(det._controller.get_deadtime(exposure) for det in detectors) # noqa: SLF001 trigger_info = TriggerInfo( number_of_triggers=number_of_frames * repeats, diff --git a/tests/core/test_flyer.py b/tests/core/test_flyer.py index 45ddaa8c86..edd38f6fd6 100644 --- a/tests/core/test_flyer.py +++ b/tests/core/test_flyer.py @@ -180,7 +180,7 @@ def flying_plan(): assert flyer._trigger_logic.state == TriggerState.preparing for detector in detectors: - detector.controller.disarm.assert_called_once() # type: ignore + detector._controller.disarm.assert_called_once() # type: ignore yield from bps.open_run() yield from bps.declare_stream(*detectors, name="main_stream", collect=True) @@ -199,8 +199,8 @@ def flying_plan(): # Manually increment the index as if a frame was taken frames_completed += frames for detector in detectors: - yield from bps.abs_set(detector.writer.dummy_signal, frames_completed) - detector.writer.index = frames_completed + yield from bps.abs_set(detector._writer.dummy_signal, frames_completed) + detector._writer.index = frames_completed done = False while not done: try: @@ -225,7 +225,7 @@ def flying_plan(): yield from bps.unstage_all(flyer, *detectors) for detector in detectors: - assert detector.controller.disarm.called # type: ignore + assert detector._controller.disarm.called # type: ignore assert trigger_logic.state == TriggerState.stopping # fly scan @@ -320,9 +320,9 @@ def flying_plan(): # Manually increment the index as if a frame was taken for detector in detectors: yield from bps.abs_set( - detector.writer.dummy_signal, trigger_info.total_number_of_triggers + detector._writer.dummy_signal, trigger_info.total_number_of_triggers ) - detector.writer.index = trigger_info.total_number_of_triggers + detector._writer.index = trigger_info.total_number_of_triggers yield from bps.wait(group="complete") @@ -338,7 +338,7 @@ def flying_plan(): assert detector._completable_frames == 0 assert detector._frames_to_complete == 0 assert detector._number_of_triggers_iter is None - assert detector.controller.wait_for_idle.called # type: ignore + assert detector._controller.wait_for_idle.called # type: ignore # This is an additional kickoff # Ensuring stop iteration is called if kickoff is invoked after complete diff --git a/tests/epics/adaravis/test_aravis.py b/tests/epics/adaravis/test_aravis.py index 270b661e55..296f94609b 100644 --- a/tests/epics/adaravis/test_aravis.py +++ b/tests/epics/adaravis/test_aravis.py @@ -13,7 +13,7 @@ @pytest.fixture def test_adaravis(ad_standard_det_factory) -> adaravis.AravisDetector: - return ad_standard_det_factory(adaravis.AravisDetector) + return ad_standard_det_factory(adaravis.AravisController) @pytest.mark.parametrize("exposure_time", [0.0, 0.1, 1.0, 10.0, 100.0]) @@ -21,14 +21,14 @@ async def test_deadtime_invariant_with_exposure_time( exposure_time: float, test_adaravis: adaravis.AravisDetector, ): - assert test_adaravis.controller.get_deadtime(exposure_time) == 1961e-6 + assert test_adaravis._controller.get_deadtime(exposure_time) == 1961e-6 async def test_trigger_source_set_to_gpio_line(test_adaravis: adaravis.AravisDetector): set_mock_value(test_adaravis.drv.trigger_source, "Freerun") async def trigger_and_complete(): - await test_adaravis.controller.prepare( + await test_adaravis._controller.prepare( TriggerInfo( number_of_triggers=1, trigger=DetectorTrigger.edge_trigger, @@ -42,7 +42,7 @@ async def trigger_and_complete(): # Default TriggerSource assert (await test_adaravis.drv.trigger_source.get_value()) == "Freerun" - test_adaravis.set_external_trigger_gpio(1) + test_adaravis._controller.set_external_trigger_gpio(1) # TriggerSource not changed by setting gpio assert (await test_adaravis.drv.trigger_source.get_value()) == "Freerun" @@ -51,24 +51,24 @@ async def trigger_and_complete(): # TriggerSource changes assert (await test_adaravis.drv.trigger_source.get_value()) == "Line1" - test_adaravis.set_external_trigger_gpio(3) + test_adaravis._controller.set_external_trigger_gpio(3) # TriggerSource not changed by setting gpio await trigger_and_complete() assert (await test_adaravis.drv.trigger_source.get_value()) == "Line3" def test_gpio_pin_limited(test_adaravis: adaravis.AravisDetector): - assert test_adaravis.get_external_trigger_gpio() == 1 - test_adaravis.set_external_trigger_gpio(2) - assert test_adaravis.get_external_trigger_gpio() == 2 + assert test_adaravis._controller.get_external_trigger_gpio() == 1 + test_adaravis._controller.set_external_trigger_gpio(2) + assert test_adaravis._controller.get_external_trigger_gpio() == 2 with pytest.raises( ValueError, match=re.escape( - "AravisDetector only supports the following GPIO indices: " + "AravisController only supports the following GPIO indices: " "(1, 2, 3, 4) but was asked to use 55" ), ): - test_adaravis.set_external_trigger_gpio(55) # type: ignore + test_adaravis._controller.set_external_trigger_gpio(55) # type: ignore async def test_hints_from_hdf_writer(test_adaravis: adaravis.AravisDetector): @@ -89,7 +89,7 @@ async def test_decribe_describes_writer_dataset( assert await test_adaravis.describe() == { "test_adaravis1": { "source": "mock+ca://ARAVIS1:HDF1:FullFileName_RBV", - "shape": (10, 10), + "shape": [10, 10], "dtype": "array", "dtype_numpy": "|i1", "external": "STREAM:", @@ -135,7 +135,7 @@ async def test_can_decribe_collect( assert (await test_adaravis.describe_collect()) == { "test_adaravis1": { "source": "mock+ca://ARAVIS1:HDF1:FullFileName_RBV", - "shape": (10, 10), + "shape": [10, 10], "dtype": "array", "dtype_numpy": "|i1", "external": "STREAM:", diff --git a/tests/epics/adcore/test_drivers.py b/tests/epics/adcore/test_drivers.py index 86a5aba144..0171db2d0b 100644 --- a/tests/epics/adcore/test_drivers.py +++ b/tests/epics/adcore/test_drivers.py @@ -53,8 +53,8 @@ async def test_set_exposure_time_and_acquire_period_if_supplied_uses_deadtime( expected_acquire_period: float, ): await controller.set_exposure_time_and_acquire_period_if_supplied(exposure) - actual_exposure = await controller.driver.acquire_time.get_value() - actual_acquire_period = await controller.driver.acquire_period.get_value() + actual_exposure = await controller._driver.acquire_time.get_value() + actual_acquire_period = await controller._driver.acquire_period.get_value() assert expected_exposure == actual_exposure assert expected_acquire_period == actual_acquire_period @@ -62,7 +62,7 @@ async def test_set_exposure_time_and_acquire_period_if_supplied_uses_deadtime( async def test_start_acquiring_driver_and_ensure_status_flags_immediate_failure( controller: adcore.ADBaseController, ): - set_mock_value(controller.driver.detector_state, adcore.DetectorState.Error) + set_mock_value(controller._driver.detector_state, adcore.DetectorState.Error) acquiring = await controller.start_acquiring_driver_and_ensure_status() with pytest.raises(ValueError): await acquiring @@ -76,12 +76,12 @@ async def test_start_acquiring_driver_and_ensure_status_fails_after_some_time( Real world application; it takes some time to start acquiring, and during that time the detector gets itself into a bad state. """ - set_mock_value(controller.driver.detector_state, adcore.DetectorState.Idle) + set_mock_value(controller._driver.detector_state, adcore.DetectorState.Idle) async def wait_then_fail(): await asyncio.sleep(0) set_mock_value( - controller.driver.detector_state, adcore.DetectorState.Disconnected + controller._driver.detector_state, adcore.DetectorState.Disconnected ) controller.frame_timeout = 0.1 diff --git a/tests/epics/adcore/test_scans.py b/tests/epics/adcore/test_scans.py index bc32f8f7b5..aa5c7120eb 100644 --- a/tests/epics/adcore/test_scans.py +++ b/tests/epics/adcore/test_scans.py @@ -6,7 +6,7 @@ import bluesky.plan_stubs as bps import bluesky.plans as bp import pytest -from bluesky import RunEngine +from bluesky.run_engine import RunEngine from ophyd_async.core import ( AsyncStatus, @@ -70,6 +70,7 @@ def writer(RE, static_path_provider, tmp_path: Path) -> adcore.ADHDFWriter: static_path_provider, lambda: "test", AsyncMock(), + {}, ) @@ -79,8 +80,8 @@ async def test_hdf_writer_fails_on_timeout_with_stepscan( writer: adcore.ADHDFWriter, controller: adcore.ADBaseController, ): - set_mock_value(writer.hdf.file_path_exists, True) - detector: StandardDetector[Any] = StandardDetector( + set_mock_value(writer._fileio.file_path_exists, True) + detector: StandardDetector[Any, Any] = StandardDetector( controller, writer, name="detector" ) @@ -95,11 +96,9 @@ def test_hdf_writer_fails_on_timeout_with_flyscan( RE: RunEngine, writer: adcore.ADHDFWriter ): controller = DummyController() - set_mock_value(writer.hdf.file_path_exists, True) + set_mock_value(writer._fileio.file_path_exists, True) - detector: StandardDetector[TriggerInfo | None] = StandardDetector( - controller, writer - ) + detector: StandardDetector[Any, Any] = StandardDetector(controller, writer) trigger_logic = DummyTriggerLogic() flyer = StandardFlyer(trigger_logic, name="flyer") diff --git a/tests/epics/adcore/test_single_trigger.py b/tests/epics/adcore/test_single_trigger.py index 9bb5135cc0..b188cdd948 100644 --- a/tests/epics/adcore/test_single_trigger.py +++ b/tests/epics/adcore/test_single_trigger.py @@ -1,26 +1,28 @@ import bluesky.plan_stubs as bps import bluesky.plans as bp import pytest -from bluesky import RunEngine +from bluesky.run_engine import RunEngine import ophyd_async.plan_stubs as ops +from ophyd_async.core._device import DeviceCollector from ophyd_async.epics import adcore @pytest.fixture async def single_trigger_det_with_stats(): - stats = adcore.NDPluginStatsIO("PREFIX:STATS", name="stats") - det = adcore.SingleTriggerDetector( - drv=adcore.ADBaseIO("PREFIX:DRV"), - stats=stats, - read_uncached=[stats.unique_id], - name="det", - ) + async with DeviceCollector(mock=True): + stats = adcore.NDPluginStatsIO("PREFIX:STATS", name="stats") + det = adcore.SingleTriggerDetector( + drv=adcore.ADBaseIO("PREFIX:DRV"), + plugins={"stats": stats}, + read_uncached=[stats.unique_id], + name="det", + ) # Set non-default values to check they are set back # These are using set_mock_value to simulate the backend IOC being setup # in a particular way, rather than values being set by the Ophyd signals - yield det, stats + return det, stats async def test_single_trigger_det( @@ -51,6 +53,8 @@ def plan(): assert names == ["start", "descriptor", "event", "stop"] _, descriptor, event, _ = docs + print(descriptor) + print(event) assert descriptor["configuration"]["det"]["data"]["det-drv-acquire_time"] == 0.5 assert event["data"]["det-drv-array_counter"] == 1 assert event["data"]["det-stats-unique_id"] == 0 diff --git a/tests/epics/adcore/test_writers.py b/tests/epics/adcore/test_writers.py index 7c5b7edb4e..f374656157 100644 --- a/tests/epics/adcore/test_writers.py +++ b/tests/epics/adcore/test_writers.py @@ -11,6 +11,7 @@ ) from ophyd_async.core._mock_signal_utils import set_mock_value from ophyd_async.epics import adaravis, adcore, adkinetix, adpilatus, advimba +from ophyd_async.epics.adpilatus._pilatus_controller import PilatusReadoutTime from ophyd_async.epics.signal._signal import epics_signal_r from ophyd_async.plan_stubs._nd_attributes import setup_ndattributes, setup_ndstats_sum @@ -35,6 +36,7 @@ async def hdf_writer( static_path_provider, lambda: "test", DummyDatasetDescriber(), + {}, ) @@ -46,7 +48,7 @@ async def tiff_writer( tiff = adcore.NDFileIO("TIFF:") return adcore.ADTIFFWriter( - tiff, static_path_provider, lambda: "test", DummyDatasetDescriber() + tiff, static_path_provider, lambda: "test", DummyDatasetDescriber(), {} ) @@ -66,7 +68,7 @@ async def hdf_writer_with_stats( static_path_provider, lambda: "test", DummyDatasetDescriber(), - stats, + {"stats": stats}, ) @@ -78,7 +80,11 @@ async def detectors( async with DeviceCollector(mock=True): detectors.append(advimba.VimbaDetector("VIMBA:", static_path_provider)) detectors.append(adkinetix.KinetixDetector("KINETIX:", static_path_provider)) - detectors.append(adpilatus.PilatusDetector("PILATUS:", static_path_provider)) + detectors.append( + adpilatus.PilatusDetector( + "PILATUS:", static_path_provider, PilatusReadoutTime.pilatus3 + ) + ) detectors.append(adaravis.AravisDetector("ADARAVIS:", static_path_provider)) return detectors @@ -100,9 +106,9 @@ async def test_stats_describe_when_plugin_configured( hdf_writer_with_stats: adcore.ADHDFWriter, ): assert hdf_writer_with_stats._file is None - set_mock_value(hdf_writer_with_stats.hdf.file_path_exists, True) + set_mock_value(hdf_writer_with_stats._fileio.file_path_exists, True) set_mock_value( - hdf_writer_with_stats._plugins[0].nd_attributes_file, + hdf_writer_with_stats._plugins["stats"].nd_attributes_file, """ adkinetix.KinetixDetector: + return ad_standard_det_factory(adkinetix.KinetixController) async def test_get_deadtime( @@ -26,11 +26,11 @@ async def test_trigger_modes(test_adkinetix: adkinetix.KinetixDetector): set_mock_value(test_adkinetix.drv.trigger_mode, KinetixTriggerMode.internal) async def setup_trigger_mode(trig_mode: DetectorTrigger): - await test_adkinetix.controller.prepare( + await test_adkinetix._controller.prepare( TriggerInfo(number_of_triggers=1, trigger=trig_mode) ) - await test_adkinetix.controller.arm() - await test_adkinetix.controller.wait_for_idle() + await test_adkinetix._controller.arm() + await test_adkinetix._controller.wait_for_idle() # Prevent timeouts set_mock_value(test_adkinetix.drv.acquire, True) @@ -68,7 +68,7 @@ async def test_decribe_describes_writer_dataset( assert await test_adkinetix.describe() == { "test_adkinetix1": { "source": "mock+ca://KINETIX1:HDF1:FullFileName_RBV", - "shape": (10, 10), + "shape": [10, 10], "dtype": "array", "dtype_numpy": "|i1", "external": "STREAM:", @@ -115,7 +115,7 @@ async def test_can_decribe_collect( assert (await test_adkinetix.describe_collect()) == { "test_adkinetix1": { "source": "mock+ca://KINETIX1:HDF1:FullFileName_RBV", - "shape": (10, 10), + "shape": [10, 10], "dtype": "array", "dtype_numpy": "|i1", "external": "STREAM:", diff --git a/tests/epics/adpilatus/test_pilatus.py b/tests/epics/adpilatus/test_pilatus.py index 565e3bf323..1dca956e54 100644 --- a/tests/epics/adpilatus/test_pilatus.py +++ b/tests/epics/adpilatus/test_pilatus.py @@ -11,16 +11,16 @@ set_mock_value, ) from ophyd_async.epics import adcore, adpilatus -from ophyd_async.epics.adpilatus import PilatusController, PilatusDriverIO +from ophyd_async.epics.adpilatus import PilatusDriverIO @pytest.fixture def test_adpilatus(ad_standard_det_factory) -> adpilatus.PilatusDetector: - return ad_standard_det_factory(adpilatus.PilatusDetector) + return ad_standard_det_factory(adpilatus.PilatusController) async def test_deadtime_overridable(test_adpilatus: adpilatus.PilatusDetector): - pilatus_controller = cast(PilatusController, test_adpilatus.controller) + pilatus_controller = test_adpilatus._controller pilatus_controller._readout_time = adpilatus.PilatusReadoutTime.pilatus2 # deadtime invariant with exposure time @@ -30,7 +30,7 @@ async def test_deadtime_overridable(test_adpilatus: adpilatus.PilatusDetector): async def test_deadtime_invariant( test_adpilatus: adpilatus.PilatusDetector, ): - pilatus_controller = test_adpilatus.controller + pilatus_controller = test_adpilatus._controller # deadtime invariant with exposure time assert pilatus_controller.get_deadtime(0) == 0.95e-3 assert pilatus_controller.get_deadtime(500) == 0.95e-3 @@ -51,11 +51,11 @@ async def test_trigger_mode_set( ): async def trigger_and_complete(): set_mock_value(test_adpilatus.drv.armed, True) - await test_adpilatus.controller.prepare( + await test_adpilatus._controller.prepare( TriggerInfo(number_of_triggers=1, trigger=detector_trigger) ) - await test_adpilatus.controller.arm() - await test_adpilatus.controller.wait_for_idle() + await test_adpilatus._controller.arm() + await test_adpilatus._controller.wait_for_idle() await _trigger(test_adpilatus, expected_trigger_mode, trigger_and_complete) @@ -64,11 +64,11 @@ async def test_trigger_mode_set_without_armed_pv( test_adpilatus: adpilatus.PilatusDetector, ): async def trigger_and_complete(): - await test_adpilatus.controller.prepare( + await test_adpilatus._controller.prepare( TriggerInfo(number_of_triggers=1, trigger=DetectorTrigger.internal) ) - await test_adpilatus.controller.arm() - await test_adpilatus.controller.wait_for_idle() + await test_adpilatus._controller.arm() + await test_adpilatus._controller.wait_for_idle() with patch( "ophyd_async.epics.adpilatus._pilatus_controller.DEFAULT_TIMEOUT", @@ -125,7 +125,7 @@ async def test_exposure_time_and_acquire_period_set( async def dummy_open(multiplier: int = 0): return {} - test_adpilatus.writer.open = dummy_open + test_adpilatus._writer.open = dummy_open set_mock_value(test_adpilatus.drv.armed, True) await test_adpilatus.prepare( TriggerInfo( @@ -140,8 +140,8 @@ async def dummy_open(multiplier: int = 0): async def test_pilatus_controller(test_adpilatus: adpilatus.PilatusDetector): - pilatus = test_adpilatus.controller - pilatus_driver = cast(PilatusDriverIO, pilatus.driver) + pilatus = test_adpilatus._controller + pilatus_driver = test_adpilatus.drv set_mock_value(pilatus_driver.armed, True) await pilatus.prepare( TriggerInfo(number_of_triggers=1, trigger=DetectorTrigger.constant_gate) diff --git a/tests/epics/adsimdetector/test_sim.py b/tests/epics/adsimdetector/test_sim.py index 77dfc1d3db..df11930277 100644 --- a/tests/epics/adsimdetector/test_sim.py +++ b/tests/epics/adsimdetector/test_sim.py @@ -26,24 +26,28 @@ @pytest.fixture -def test_adsimdetector(ad_standard_det_factory): - return ad_standard_det_factory(adsimdetector.SimDetector) +def test_adsimdetector(ad_standard_det_factory: Callable) -> adsimdetector.SimDetector: + return ad_standard_det_factory(adsimdetector.SimController) @pytest.fixture -def test_adsimdetector_tiff(ad_standard_det_factory): - return ad_standard_det_factory(adsimdetector.SimDetectorTIFF) +def test_adsimdetector_tiff( + ad_standard_det_factory: Callable, +) -> adsimdetector.SimDetector: + return ad_standard_det_factory(adsimdetector.SimController, adcore.ADTIFFWriter) @pytest.fixture -def two_test_adsimdetectors(ad_standard_det_factory): - deta = ad_standard_det_factory(adsimdetector.SimDetector) - detb = ad_standard_det_factory(adsimdetector.SimDetector, number=2) +def two_test_adsimdetectors( + ad_standard_det_factory: Callable, +) -> Sequence[adsimdetector.SimDetector]: + deta = ad_standard_det_factory(adsimdetector.SimController) + detb = ad_standard_det_factory(adsimdetector.SimController, number=2) return deta, detb -def count_sim(dets: Sequence[adcore.AreaDetector], times: int = 1): +def count_sim(dets: Sequence[adsimdetector.SimDetector], times: int = 1): """Test plan to do the equivalent of bp.count for a sim detector.""" yield from bps.stage_all(*dets) @@ -51,9 +55,7 @@ def count_sim(dets: Sequence[adcore.AreaDetector], times: int = 1): for _ in range(times): read_values = {} for det in dets: - read_values[det] = yield from bps.rd( - cast(adcore.ADWriter, det.writer).fileio.num_captured - ) + read_values[det] = yield from bps.rd(det._writer._fileio.num_captured) for det in dets: yield from bps.trigger(det, wait=False, group="wait_for_trigger") @@ -61,7 +63,7 @@ def count_sim(dets: Sequence[adcore.AreaDetector], times: int = 1): yield from bps.sleep(0.2) [ set_mock_value( - cast(adcore.ADWriter, det.writer).fileio.num_captured, + det._writer._fileio.num_captured, read_values[det] + 1, ) for det in dets @@ -80,7 +82,7 @@ def count_sim(dets: Sequence[adcore.AreaDetector], times: int = 1): async def test_two_detectors_fly_different_rate( - two_test_adsimdetectors: list[adsimdetector.SimDetector], RE: RunEngine + two_test_adsimdetectors: Sequence[adsimdetector.SimDetector], RE: RunEngine ): trigger_info = TriggerInfo( number_of_triggers=15, @@ -112,23 +114,23 @@ def fly_plan(): yield from bps.trigger(det, wait=False, group="trigger_cleanup") # det[0] captures 5 frames, but we do not emit a StreamDatum as det[1] has not - set_mock_value(two_test_adsimdetectors[0].hdf.num_captured, 5) + set_mock_value(two_test_adsimdetectors[0].fileio.num_captured, 5) yield from bps.collect(*two_test_adsimdetectors) assert_n_stream_datums(0) # det[0] captures 10 frames, but we do not emit a StreamDatum as det[1] has not - set_mock_value(two_test_adsimdetectors[0].hdf.num_captured, 10) + set_mock_value(two_test_adsimdetectors[0].fileio.num_captured, 10) yield from bps.collect(*two_test_adsimdetectors) assert_n_stream_datums(0) # det[1] has caught up to first 7 frames, emit streamDatum for seq_num {1,7} - set_mock_value(two_test_adsimdetectors[1].hdf.num_captured, 7) + set_mock_value(two_test_adsimdetectors[1].fileio.num_captured, 7) yield from bps.collect(*two_test_adsimdetectors) assert_n_stream_datums(2, 1, 8) for det in two_test_adsimdetectors: - set_mock_value(det.hdf.num_captured, 15) + set_mock_value(det.fileio.num_captured, 15) # emits stream datum for seq_num {8, 15} yield from bps.collect(*two_test_adsimdetectors) @@ -152,13 +154,13 @@ async def test_two_detectors_step( RE.subscribe(lambda name, _: names.append(name)) RE.subscribe(lambda _, doc: docs.append(doc)) [ - set_mock_value(cast(adcore.ADHDFWriter, det._writer).hdf.file_path_exists, True) + set_mock_value(det._writer._fileio.file_path_exists, True) for det in two_test_adsimdetectors ] - controller_a = cast(adcore.ADBaseController, two_test_adsimdetectors[0].controller) - writer_a = cast(adcore.ADHDFWriter, two_test_adsimdetectors[0].writer) - writer_b = cast(adcore.ADHDFWriter, two_test_adsimdetectors[1].writer) + controller_a = two_test_adsimdetectors[0]._controller + writer_a = two_test_adsimdetectors[0]._writer + writer_b = two_test_adsimdetectors[1]._writer info_a = writer_a._path_provider(device_name=writer_a._name_provider()) info_b = writer_b._path_provider(device_name=writer_b._name_provider()) file_name_a = None @@ -168,22 +170,26 @@ def plan(): nonlocal file_name_a, file_name_b yield from count_sim(two_test_adsimdetectors, times=1) - drv = controller_a.driver + drv = controller_a._driver assert False is (yield from bps.rd(drv.acquire)) assert adcore.ImageMode.multiple == (yield from bps.rd(drv.image_mode)) - hdfb = writer_b.hdf + hdfb = cast(adcore.NDFileHDFIO, writer_b._fileio) assert True is (yield from bps.rd(hdfb.lazy_open)) assert True is (yield from bps.rd(hdfb.swmr_mode)) assert 0 == (yield from bps.rd(hdfb.num_capture)) assert adcore.FileWriteMode.stream == (yield from bps.rd(hdfb.file_write_mode)) - assert (yield from bps.rd(writer_a.hdf.file_path)) == str(info_a.directory_path) - file_name_a = yield from bps.rd(writer_a.hdf.file_name) + assert (yield from bps.rd(writer_a._fileio.file_path)) == str( + info_a.directory_path + ) + file_name_a = yield from bps.rd(writer_a._fileio.file_name) assert file_name_a == info_a.filename - assert (yield from bps.rd(writer_b.hdf.file_path)) == str(info_b.directory_path) - file_name_b = yield from bps.rd(writer_b.hdf.file_name) + assert (yield from bps.rd(writer_b._fileio.file_path)) == str( + info_b.directory_path + ) + file_name_b = yield from bps.rd(writer_b._fileio.file_name) assert file_name_b == info_b.filename RE(plan()) @@ -205,8 +211,8 @@ def plan(): assert descriptor["configuration"]["test_adsim2"]["data"][ "test_adsim2-drv-acquire_time" ] == pytest.approx(1.8) - assert descriptor["data_keys"]["test_adsim1"]["shape"] == (10, 10) - assert descriptor["data_keys"]["test_adsim2"]["shape"] == (11, 11) + assert descriptor["data_keys"]["test_adsim1"]["shape"] == [10, 10] + assert descriptor["data_keys"]["test_adsim2"]["shape"] == [11, 11] assert sda["stream_resource"] == sra["uid"] assert sdb["stream_resource"] == srb["uid"] assert ( @@ -221,40 +227,40 @@ def plan(): assert event["data"] == {} -@pytest.mark.parametrize( - "detector_class", [adsimdetector.SimDetector, adsimdetector.SimDetectorTIFF] -) +@pytest.mark.parametrize("writer_cls", [adcore.ADHDFWriter, adcore.ADTIFFWriter]) async def test_detector_writes_to_file( RE: RunEngine, ad_standard_det_factory: Callable, - detector_class: type[adsimdetector.SimDetector], + writer_cls: type[adcore.ADWriter], tmp_path: Path, ): - test_adsimdetector = ad_standard_det_factory(detector_class) + test_adsimdetector: adsimdetector.SimDetector = ad_standard_det_factory( + adsimdetector.SimController, writer_cls + ) names = [] docs = [] RE.subscribe(lambda name, _: names.append(name)) RE.subscribe(lambda _, doc: docs.append(doc)) set_mock_value( - cast(adcore.ADHDFWriter, test_adsimdetector.writer).fileio.file_path_exists, + test_adsimdetector._writer._fileio.file_path_exists, True, ) RE(count_sim([test_adsimdetector], times=3)) - assert await cast( - adcore.ADWriter, test_adsimdetector.writer - ).fileio.file_path.get_value() == str(tmp_path) + assert await test_adsimdetector._writer._fileio.file_path.get_value() == str( + tmp_path + ) descriptor_index = names.index("descriptor") assert docs[descriptor_index].get("data_keys").get(test_adsimdetector.name).get( "shape" - ) == ( + ) == [ 10, 10, - ) + ] assert names == [ "start", "descriptor", @@ -342,7 +348,7 @@ def test_detector_with_unnamed_or_disconnected_config_sigs( ): dp = StaticPathProvider(static_filename_provider, tmp_path) - some_other_driver = adcore.ADBaseIO("TEST", name=driver_name) + some_other_driver = adsimdetector.SimDriverIO("TEST", name=driver_name) det = adsimdetector.SimDetector( "FOO:", @@ -380,7 +386,7 @@ async def test_ad_sim_controller(test_adsimdetector: adsimdetector.SimDetector): await ad.arm() await ad.wait_for_idle() - driver = ad.driver + driver = ad._driver assert await driver.num_images.get_value() == 1 assert await driver.image_mode.get_value() == adcore.ImageMode.multiple assert await driver.acquire.get_value() is True diff --git a/tests/epics/advimba/test_vimba.py b/tests/epics/advimba/test_vimba.py index 25ec7f2328..940b66f52b 100644 --- a/tests/epics/advimba/test_vimba.py +++ b/tests/epics/advimba/test_vimba.py @@ -6,7 +6,7 @@ set_mock_value, ) from ophyd_async.core._detector import TriggerInfo -from ophyd_async.epics import advimba +from ophyd_async.epics import adcore, advimba from ophyd_async.epics.advimba._vimba_io import ( VimbaExposeOutMode, VimbaOnOff, @@ -16,7 +16,7 @@ @pytest.fixture def test_advimba(ad_standard_det_factory) -> advimba.VimbaDetector: - return ad_standard_det_factory(advimba.VimbaDetector) + return ad_standard_det_factory(advimba.VimbaController, adcore.ADHDFWriter) async def test_get_deadtime( @@ -32,11 +32,11 @@ async def test_arming_trig_modes(test_advimba: advimba.VimbaDetector): set_mock_value(test_advimba.drv.exposure_mode, VimbaExposeOutMode.timed) async def setup_trigger_mode(trig_mode: DetectorTrigger): - await test_advimba.controller.prepare( + await test_advimba._controller.prepare( TriggerInfo(number_of_triggers=1, trigger=trig_mode) ) - await test_advimba.controller.arm() - await test_advimba.controller.wait_for_idle() + await test_advimba._controller.arm() + await test_advimba._controller.wait_for_idle() # Prevent timeouts set_mock_value(test_advimba.drv.acquire, True) @@ -84,7 +84,7 @@ async def test_decribe_describes_writer_dataset( assert await test_advimba.describe() == { "test_advimba1": { "source": "mock+ca://VIMBA1:HDF1:FullFileName_RBV", - "shape": (10, 10), + "shape": [10, 10], "dtype": "array", "dtype_numpy": "|i1", "external": "STREAM:", @@ -131,7 +131,7 @@ async def test_can_decribe_collect( assert (await test_advimba.describe_collect()) == { "test_advimba1": { "source": "mock+ca://VIMBA1:HDF1:FullFileName_RBV", - "shape": (10, 10), + "shape": [10, 10], "dtype": "array", "dtype_numpy": "|i1", "external": "STREAM:", diff --git a/tests/epics/conftest.py b/tests/epics/conftest.py index a7cf0f05a6..d8ad248085 100644 --- a/tests/epics/conftest.py +++ b/tests/epics/conftest.py @@ -7,57 +7,69 @@ from ophyd_async.core._device import DeviceCollector from ophyd_async.core._mock_signal_utils import callback_on_mock_put, set_mock_value from ophyd_async.epics import adcore +from ophyd_async.epics.adcore._core_detector import AreaDetector +from ophyd_async.epics.adcore._core_logic import ADBaseController +from ophyd_async.epics.adcore._core_writer import ADWriter @pytest.fixture def ad_standard_det_factory( RE: RunEngine, static_path_provider, -) -> Callable[[type[adcore.AreaDetector], int], adcore.AreaDetector]: +) -> Callable[[type[ADBaseController], type[ADWriter], int], AreaDetector]: def generate_ad_standard_det( - ad_standard_detector_class: type[adcore.AreaDetector], number=1 - ) -> adcore.AreaDetector: - # Dynamically generate a name based on the class of detector - detector_name = ad_standard_detector_class.__name__ - if detector_name.endswith("Detector"): - detector_name = detector_name[: -len("Detector")] - elif detector_name.endswith("DetectorTIFF"): - detector_name = ( - detector_name.split("Detector")[0] - + "_" - + detector_name.split("Detector")[1] - ) + controller_cls: type[ADBaseController], + writer_cls: type[adcore.ADWriter] = adcore.ADHDFWriter, + number=1, + ) -> AreaDetector: + # Dynamically generate a name based on the class of controller + detector_name = controller_cls.__name__ + if detector_name.endswith("Controller"): + detector_name = detector_name[: -len("Controller")] with DeviceCollector(mock=True): - test_adstandard_det = ad_standard_detector_class( - f"{detector_name.upper()}{number}:", + prefix = f"{detector_name.upper()}{number}:" + name = f"test_ad{detector_name.lower()}{number}" + + controller, driver = controller_cls.controller_and_drv( + prefix + "cam1:", name=name + ) + + test_adstandard_det = AreaDetector[controller_cls, writer_cls]( + prefix, + driver, + controller, + writer_cls, static_path_provider, - name=f"test_ad{detector_name.lower()}{number}", + {}, + name=name, ) def on_set_file_path_callback(value, **kwargs): if os.path.exists(value): - set_mock_value(test_adstandard_det.writer.fileio.file_path_exists, True) set_mock_value( - test_adstandard_det.writer.fileio.full_file_name, - f"{value}/{static_path_provider._filename_provider(device_name=test_adstandard_det.name)}{test_adstandard_det.writer._file_extension}", + test_adstandard_det._writer._fileio.file_path_exists, True + ) + set_mock_value( + test_adstandard_det._writer._fileio.full_file_name, + f"{value}/{static_path_provider._filename_provider(device_name=test_adstandard_det.name)}{test_adstandard_det._writer._file_extension}", ) callback_on_mock_put( - test_adstandard_det.writer.fileio.file_path, on_set_file_path_callback + test_adstandard_det._writer._fileio.file_path, on_set_file_path_callback ) # Set some sensible defaults to mimic a real detector setup set_mock_value(test_adstandard_det.drv.acquire_time, (number - 0.2)) set_mock_value(test_adstandard_det.drv.acquire_period, float(number)) - set_mock_value(test_adstandard_det.writer.fileio.capture, True) + set_mock_value(test_adstandard_det._writer._fileio.capture, True) # Set number of frames per chunk and frame dimensions to something reasonable set_mock_value(test_adstandard_det.drv.array_size_x, (9 + number)) set_mock_value(test_adstandard_det.drv.array_size_y, (9 + number)) - if isinstance(test_adstandard_det.writer, adcore.ADHDFWriter): - set_mock_value(test_adstandard_det.writer.hdf.num_frames_chunks, 1) + if isinstance(test_adstandard_det.fileio, adcore.NDFileHDFIO): + set_mock_value(test_adstandard_det.fileio.num_frames_chunks, 1) return test_adstandard_det diff --git a/tests/epics/eiger/test_eiger_detector.py b/tests/epics/eiger/test_eiger_detector.py index 4ad6557ee0..a637e7a980 100644 --- a/tests/epics/eiger/test_eiger_detector.py +++ b/tests/epics/eiger/test_eiger_detector.py @@ -21,7 +21,7 @@ def test_when_detector_initialised_then_driver_and_odin_have_expected_prefixes( async def test_when_prepared_with_energy_then_energy_set_on_detector(detector): - detector.controller.arm = AsyncMock() + detector._controller.arm = AsyncMock() await detector.prepare( EigerTriggerInfo( frame_timeout=None, diff --git a/tests/fastcs/panda/test_hdf_panda.py b/tests/fastcs/panda/test_hdf_panda.py index 7fcf78b7e3..6c465868ab 100644 --- a/tests/fastcs/panda/test_hdf_panda.py +++ b/tests/fastcs/panda/test_hdf_panda.py @@ -69,8 +69,8 @@ def check_dir_exits(value, **kwargs): async def test_hdf_panda_passes_blocks_to_controller(mock_hdf_panda: HDFPanda): - assert hasattr(mock_hdf_panda.controller, "pcap") - assert mock_hdf_panda.controller.pcap is mock_hdf_panda.pcap + assert hasattr(mock_hdf_panda._controller, "pcap") + assert mock_hdf_panda._controller.pcap is mock_hdf_panda.pcap async def test_hdf_panda_hardware_triggered_flyable( @@ -137,7 +137,7 @@ def flying_plan(): # Verify that _completable_frames is reset to 0 after the final complete. assert mock_hdf_panda._completable_frames == 0 yield from bps.unstage_all(flyer, mock_hdf_panda) - yield from bps.wait_for([lambda: mock_hdf_panda.controller.disarm()]) + yield from bps.wait_for([lambda: mock_hdf_panda._controller.disarm()]) # fly scan RE(flying_plan()) diff --git a/tests/plan_stubs/test_fly.py b/tests/plan_stubs/test_fly.py index 4181da1c3a..6533ff6e5c 100644 --- a/tests/plan_stubs/test_fly.py +++ b/tests/plan_stubs/test_fly.py @@ -117,8 +117,8 @@ def __init__( async def complete(self): assert self._trigger_info assert self._fly_start - self.writer.increment_index() - async for index in self.writer.observe_indices_written( + self._writer.increment_index() + async for index in self._writer.observe_indices_written( self._trigger_info.frame_timeout or ( DEFAULT_TIMEOUT @@ -268,7 +268,7 @@ def flying_plan(): ) for detector in detector_list: - detector.controller.disarm.assert_called_once() # type: ignore + detector._controller.disarm.assert_called_once() # type: ignore yield from bps.open_run() yield from bps.declare_stream(*detector_list, name="main_stream", collect=True) @@ -285,7 +285,7 @@ def flying_plan(): # Manually incremenet the index as if a frame was taken for detector in detector_list: - detector.writer.increment_index() + detector._writer.increment_index() set_mock_value(flyer.trigger_logic.seq.active, 0) @@ -307,7 +307,7 @@ def flying_plan(): yield from bps.unstage_all(flyer, *detector_list) for detector in detector_list: - assert detector.controller.disarm.called # type: ignore + assert detector._controller.disarm.called # type: ignore # fly scan RE(flying_plan()) @@ -432,4 +432,6 @@ def fly(): RE(fly()) for detector in detectors: - assert detector.writer.observe_indices_written_timeout_log == [expected_timeout] + assert detector._writer.observe_indices_written_timeout_log == [ + expected_timeout + ] diff --git a/tests/sim/test_sim_detector.py b/tests/sim/test_sim_detector.py index 7631a5b558..0290ff9d92 100644 --- a/tests/sim/test_sim_detector.py +++ b/tests/sim/test_sim_detector.py @@ -21,8 +21,8 @@ async def test_sim_pattern_detector_initialization( async def test_detector_creates_controller_and_writer( sim_pattern_detector: PatternDetector, ): - assert sim_pattern_detector.writer - assert sim_pattern_detector.controller + assert sim_pattern_detector._writer + assert sim_pattern_detector._controller def test_writes_pattern_to_file( diff --git a/tests/sim/test_streaming_plan.py b/tests/sim/test_streaming_plan.py index 222f92d685..d369a0d518 100644 --- a/tests/sim/test_streaming_plan.py +++ b/tests/sim/test_streaming_plan.py @@ -38,7 +38,7 @@ def plan(): "event", "stop", ] - await sim_pattern_detector.writer.close() + await sim_pattern_detector._writer.close() async def test_plan(RE: RunEngine, sim_pattern_detector: PatternDetector): From 6dc09f3c74eff3cc43dc3e849fe850bc80ff7ec7 Mon Sep 17 00:00:00 2001 From: Jakub Wlodek Date: Fri, 22 Nov 2024 15:26:10 -0500 Subject: [PATCH 17/60] Revert change in test that was resolved by pydantic version update --- tests/fastcs/panda/test_trigger.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fastcs/panda/test_trigger.py b/tests/fastcs/panda/test_trigger.py index 66d6e1c5f3..f12b5f6051 100644 --- a/tests/fastcs/panda/test_trigger.py +++ b/tests/fastcs/panda/test_trigger.py @@ -145,7 +145,7 @@ def full_seq_table(trigger): full_seq_table(["A"]) assert ( "Input should be 'Immediate', 'BITA=0', 'BITA=1', 'BITB=0', 'BITB=1', " - "'BITC...' [type=enum, input_value='A', input_type=str]" + "'BITC... [type=enum, input_value='A', input_type=str]" ) in str(exc) # Pydantic is able to infer type from these From 1f7dcd737904f42692219c3307c8dfe0f55cd19f Mon Sep 17 00:00:00 2001 From: Jakub Wlodek Date: Fri, 22 Nov 2024 15:30:32 -0500 Subject: [PATCH 18/60] Remove debugging prints --- src/ophyd_async/core/_detector.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/ophyd_async/core/_detector.py b/src/ophyd_async/core/_detector.py index 1844f76a3c..61ae9d5edb 100644 --- a/src/ophyd_async/core/_detector.py +++ b/src/ophyd_async/core/_detector.py @@ -270,7 +270,6 @@ async def describe(self) -> dict[str, DataKey]: @AsyncStatus.wrap async def trigger(self) -> None: - print("In trigger") if self._trigger_info is None: await self.prepare( TriggerInfo( @@ -281,7 +280,6 @@ async def trigger(self) -> None: frame_timeout=None, ) ) - print(self._trigger_info) assert self._trigger_info assert self._trigger_info.trigger is DetectorTrigger.internal # Arm the detector and wait for it to finish. From 35dd1b16179f676bdc9bd8b0e5586e88d9940a12 Mon Sep 17 00:00:00 2001 From: Jakub Wlodek Date: Fri, 22 Nov 2024 15:58:25 -0500 Subject: [PATCH 19/60] Linter fixes --- src/ophyd_async/epics/adaravis/_aravis.py | 1 + src/ophyd_async/epics/adcore/_core_io.py | 4 +-- .../epics/adpilatus/_pilatus_controller.py | 1 - tests/epics/adcore/test_single_trigger.py | 2 +- tests/epics/adcore/test_writers.py | 3 +- tests/epics/advimba/test_vimba.py | 3 +- tests/epics/conftest.py | 7 +++-- tests/epics/signal/test_signals.py | 30 +++++++++---------- 8 files changed, 24 insertions(+), 27 deletions(-) diff --git a/src/ophyd_async/epics/adaravis/_aravis.py b/src/ophyd_async/epics/adaravis/_aravis.py index 7cbff38afd..8dafd5c958 100644 --- a/src/ophyd_async/epics/adaravis/_aravis.py +++ b/src/ophyd_async/epics/adaravis/_aravis.py @@ -3,6 +3,7 @@ from ophyd_async.core import PathProvider from ophyd_async.core._signal import SignalR from ophyd_async.epics import adcore + from ._aravis_controller import AravisController diff --git a/src/ophyd_async/epics/adcore/_core_io.py b/src/ophyd_async/epics/adcore/_core_io.py index 19a8fda0d0..1eedcd6326 100644 --- a/src/ophyd_async/epics/adcore/_core_io.py +++ b/src/ophyd_async/epics/adcore/_core_io.py @@ -1,9 +1,7 @@ import asyncio -from enum import Enum -from ophyd_async.core import Device -from ophyd_async.core._providers import DatasetDescriber from ophyd_async.core import Device, StrictEnum +from ophyd_async.core._providers import DatasetDescriber from ophyd_async.epics.core import ( epics_signal_r, epics_signal_rw, diff --git a/src/ophyd_async/epics/adpilatus/_pilatus_controller.py b/src/ophyd_async/epics/adpilatus/_pilatus_controller.py index e060e27e36..7629651219 100644 --- a/src/ophyd_async/epics/adpilatus/_pilatus_controller.py +++ b/src/ophyd_async/epics/adpilatus/_pilatus_controller.py @@ -8,7 +8,6 @@ TriggerInfo, wait_for_value, ) -from ophyd_async.core import TriggerInfo from ophyd_async.epics import adcore from ._pilatus_io import PilatusDriverIO, PilatusTriggerMode diff --git a/tests/epics/adcore/test_single_trigger.py b/tests/epics/adcore/test_single_trigger.py index 7c427089ac..f16a7f5fb1 100644 --- a/tests/epics/adcore/test_single_trigger.py +++ b/tests/epics/adcore/test_single_trigger.py @@ -4,7 +4,7 @@ from bluesky.run_engine import RunEngine import ophyd_async.plan_stubs as ops -from ophyd_async.core._device import DeviceCollector +from ophyd_async.core import DeviceCollector from ophyd_async.epics import adcore diff --git a/tests/epics/adcore/test_writers.py b/tests/epics/adcore/test_writers.py index d69a542118..073b3e8d47 100644 --- a/tests/epics/adcore/test_writers.py +++ b/tests/epics/adcore/test_writers.py @@ -12,8 +12,7 @@ set_mock_value, ) from ophyd_async.epics import adaravis, adcore, adkinetix, adpilatus, advimba -from ophyd_async.epics.adpilatus._pilatus_controller import PilatusReadoutTime -from ophyd_async.plan_stubs._nd_attributes import setup_ndattributes, setup_ndstats_sum +from ophyd_async.epics.adpilatus import PilatusReadoutTime from ophyd_async.epics.core import epics_signal_r from ophyd_async.plan_stubs import setup_ndattributes, setup_ndstats_sum diff --git a/tests/epics/advimba/test_vimba.py b/tests/epics/advimba/test_vimba.py index 2350732d20..28923c2003 100644 --- a/tests/epics/advimba/test_vimba.py +++ b/tests/epics/advimba/test_vimba.py @@ -6,9 +6,8 @@ TriggerInfo, set_mock_value, ) -from ophyd_async.core._detector import TriggerInfo from ophyd_async.epics import adcore, advimba -from ophyd_async.epics.advimba._vimba_io import ( +from ophyd_async.epics.advimba import ( VimbaExposeOutMode, VimbaOnOff, VimbaTriggerSource, diff --git a/tests/epics/conftest.py b/tests/epics/conftest.py index 839a5b1e32..f089128c6e 100644 --- a/tests/epics/conftest.py +++ b/tests/epics/conftest.py @@ -4,8 +4,7 @@ import pytest from bluesky.run_engine import RunEngine -from ophyd_async.core._device import DeviceCollector -from ophyd_async.core._mock_signal_utils import callback_on_mock_put, set_mock_value +from ophyd_async.core import DeviceCollector, callback_on_mock_put, set_mock_value from ophyd_async.epics import adcore @@ -13,7 +12,9 @@ def ad_standard_det_factory( RE: RunEngine, static_path_provider, -) -> Callable[[type[adcore.ADBaseController], type[adcore.ADWriter], int], adcore.AreaDetector]: +) -> Callable[ + [type[adcore.ADBaseController], type[adcore.ADWriter], int], adcore.AreaDetector +]: def generate_ad_standard_det( controller_cls: type[adcore.ADBaseController], writer_cls: type[adcore.ADWriter] = adcore.ADHDFWriter, diff --git a/tests/epics/signal/test_signals.py b/tests/epics/signal/test_signals.py index 2b7fea53a4..07e8a21950 100644 --- a/tests/epics/signal/test_signals.py +++ b/tests/epics/signal/test_signals.py @@ -908,25 +908,25 @@ async def test_bool_works_for_mismatching_enums(ioc, protocol): await sig.connect() -# @pytest.mark.skipif(os.name == "nt", reason="Hangs on windows for unknown reasons") -# @PARAMETERISE_PROTOCOLS -# async def test_can_read_using_ophyd_async_then_ophyd(ioc, protocol): -# prefix = get_prefix(ioc, protocol) -# oa_read = f"{protocol}://{prefix}float_prec_1" -# ophyd_read = f"{prefix}float_prec_0" +@pytest.mark.skipif(os.name == "nt", reason="Hangs on windows for unknown reasons") +@PARAMETERISE_PROTOCOLS +async def test_can_read_using_ophyd_async_then_ophyd(ioc, protocol): + prefix = get_prefix(ioc, protocol) + oa_read = f"{protocol}://{prefix}float_prec_1" + ophyd_read = f"{prefix}float_prec_0" -# ophyd_async_sig = epics_signal_rw(float, oa_read) -# await ophyd_async_sig.connect() -# ophyd_signal = EpicsSignal(ophyd_read) -# ophyd_signal.wait_for_connection(timeout=5) + ophyd_async_sig = epics_signal_rw(float, oa_read) + await ophyd_async_sig.connect() + ophyd_signal = EpicsSignal(ophyd_read) + ophyd_signal.wait_for_connection(timeout=5) -# RE = RunEngine() + RE = RunEngine() -# def my_plan(): -# yield from bps.rd(ophyd_async_sig) -# yield from bps.rd(ophyd_signal) + def my_plan(): + yield from bps.rd(ophyd_async_sig) + yield from bps.rd(ophyd_signal) -# RE(my_plan()) + RE(my_plan()) def test_signal_module_emits_deprecation_warning(): From 8112220f0081ce86bd51ba0a617270ab072027f9 Mon Sep 17 00:00:00 2001 From: Jakub Wlodek Date: Fri, 22 Nov 2024 16:04:03 -0500 Subject: [PATCH 20/60] Fix linter error --- src/ophyd_async/epics/adcore/_core_logic.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/ophyd_async/epics/adcore/_core_logic.py b/src/ophyd_async/epics/adcore/_core_logic.py index 0852384e94..3f5e7987e2 100644 --- a/src/ophyd_async/epics/adcore/_core_logic.py +++ b/src/ophyd_async/epics/adcore/_core_logic.py @@ -117,7 +117,10 @@ async def start_acquiring_driver_and_ensure_status(self) -> AsyncStatus: """ status = await set_and_wait_for_value( - self._driver.acquire, True, timeout=self.frame_timeout, wait_for_set_completion=False + self._driver.acquire, + True, + timeout=self.frame_timeout, + wait_for_set_completion=False, ) async def complete_acquisition() -> None: From ac1e509dc735c9c7190b98f98887da79d478b76d Mon Sep 17 00:00:00 2001 From: jwlodek Date: Tue, 26 Nov 2024 14:39:25 -0500 Subject: [PATCH 21/60] Move creation of writer outside of base AreaDetector class init per review --- src/ophyd_async/epics/adaravis/_aravis.py | 17 ++++++++++---- .../epics/adcore/_core_detector.py | 21 +++++------------ src/ophyd_async/epics/adcore/_core_writer.py | 11 +++++++-- src/ophyd_async/epics/adkinetix/_kinetix.py | 23 +++++++++++++++---- src/ophyd_async/epics/adpilatus/_pilatus.py | 17 ++++++++++---- src/ophyd_async/epics/adsimdetector/_sim.py | 16 +++++++++---- src/ophyd_async/epics/advimba/_vimba.py | 17 ++++++++++---- tests/epics/conftest.py | 13 +++++++---- 8 files changed, 90 insertions(+), 45 deletions(-) diff --git a/src/ophyd_async/epics/adaravis/_aravis.py b/src/ophyd_async/epics/adaravis/_aravis.py index 8dafd5c958..a274f0b354 100644 --- a/src/ophyd_async/epics/adaravis/_aravis.py +++ b/src/ophyd_async/epics/adaravis/_aravis.py @@ -3,6 +3,7 @@ from ophyd_async.core import PathProvider from ophyd_async.core._signal import SignalR from ophyd_async.epics import adcore +from ophyd_async.epics.adcore._core_io import ADBaseDatasetDescriber from ._aravis_controller import AravisController @@ -29,17 +30,23 @@ def __init__( controller, driver = AravisController.controller_and_drv( prefix + drv_suffix, gpio_number=gpio_number, name=name ) + writer, fileio = writer_cls.writer_and_io( + prefix, + path_provider, + lambda: name, + ADBaseDatasetDescriber(driver), + fileio_suffix=fileio_suffix, + plugins=plugins, + ) super().__init__( - prefix=prefix, driver=driver, controller=controller, - writer_cls=writer_cls, - fileio_suffix=fileio_suffix, - path_provider=path_provider, + fileio=fileio, + writer=writer, plugins=plugins, name=name, config_sigs=config_sigs, ) - self.drv = driver + self.fileio = fileio diff --git a/src/ophyd_async/epics/adcore/_core_detector.py b/src/ophyd_async/epics/adcore/_core_detector.py index a27971fe7b..3e7d600e5e 100644 --- a/src/ophyd_async/epics/adcore/_core_detector.py +++ b/src/ophyd_async/epics/adcore/_core_detector.py @@ -1,8 +1,8 @@ from collections.abc import Sequence -from ophyd_async.core import PathProvider, SignalR, StandardDetector +from ophyd_async.core import SignalR, StandardDetector -from ._core_io import ADBaseDatasetDescriber, ADBaseIO, NDPluginBaseIO +from ._core_io import ADBaseIO, NDFileIO, NDPluginBaseIO from ._core_logic import ADBaseControllerT from ._core_writer import ADWriterT @@ -10,25 +10,16 @@ class AreaDetector(StandardDetector[ADBaseControllerT, ADWriterT]): def __init__( self, - prefix: str, driver: ADBaseIO, controller: ADBaseControllerT, - writer_cls: type[ADWriterT], - path_provider: PathProvider, - plugins: dict[str, NDPluginBaseIO] | None, + fileio: NDFileIO, + writer: ADWriterT, + plugins: dict[str, NDPluginBaseIO] | None = None, config_sigs: Sequence[SignalR] = (), name: str = "", - fileio_suffix: str | None = None, ): self.drv = driver - writer, self.fileio = writer_cls.writer_and_io( - prefix + (fileio_suffix or writer_cls.default_suffix), - path_provider, - lambda: name, - ADBaseDatasetDescriber(self.drv), - plugins=plugins, - name=name, - ) + self.fileio = fileio if plugins is not None: for name, plugin in plugins.items(): diff --git a/src/ophyd_async/epics/adcore/_core_writer.py b/src/ophyd_async/epics/adcore/_core_writer.py index 3e73fe29d9..7aed65429b 100644 --- a/src/ophyd_async/epics/adcore/_core_writer.py +++ b/src/ophyd_async/epics/adcore/_core_writer.py @@ -70,15 +70,22 @@ def writer_and_io( path_provider: PathProvider, name_provider: NameProvider, dataset_describer: ADBaseDatasetDescriber, - plugins: dict[str, NDPluginBaseIO] | None = None, + fileio_suffix: str | None = None, name: str = "", + plugins: dict[str, NDPluginBaseIO] | None = None, ) -> tuple[ADWriterT, NDFileIOT]: try: fileio_cls = get_args(cls.__orig_bases__[0])[0] # type: ignore except IndexError as err: raise RuntimeError("File IO class for writer not specified!") from err - fileio = fileio_cls(prefix, name=name) + if fileio_suffix is None: + fileio_prefix = prefix + cls.default_suffix + else: + fileio_prefix = prefix + fileio_suffix + + fileio = fileio_cls(fileio_prefix, name=name) + writer = cls( fileio, path_provider, name_provider, dataset_describer, plugins=plugins ) diff --git a/src/ophyd_async/epics/adkinetix/_kinetix.py b/src/ophyd_async/epics/adkinetix/_kinetix.py index b2823853a5..edfbbe3b80 100644 --- a/src/ophyd_async/epics/adkinetix/_kinetix.py +++ b/src/ophyd_async/epics/adkinetix/_kinetix.py @@ -1,7 +1,13 @@ from collections.abc import Sequence from ophyd_async.core import PathProvider, SignalR -from ophyd_async.epics.adcore import ADHDFWriter, ADWriter, AreaDetector, NDPluginBaseIO +from ophyd_async.epics.adcore import ( + ADBaseDatasetDescriber, + ADHDFWriter, + ADWriter, + AreaDetector, + NDPluginBaseIO, +) from ._kinetix_controller import KinetixController @@ -26,16 +32,23 @@ def __init__( controller, driver = KinetixController.controller_and_drv( prefix + drv_suffix, name=name ) + writer, fileio = writer_cls.writer_and_io( + prefix, + path_provider, + lambda: name, + ADBaseDatasetDescriber(driver), + fileio_suffix=fileio_suffix, + plugins=plugins, + ) super().__init__( - prefix=prefix, driver=driver, controller=controller, - writer_cls=writer_cls, - path_provider=path_provider, + fileio=fileio, + writer=writer, plugins=plugins, name=name, - fileio_suffix=fileio_suffix, config_sigs=config_sigs, ) self.drv = driver + self.fileio = fileio diff --git a/src/ophyd_async/epics/adpilatus/_pilatus.py b/src/ophyd_async/epics/adpilatus/_pilatus.py index 09f4bda0b6..f736ddc106 100644 --- a/src/ophyd_async/epics/adpilatus/_pilatus.py +++ b/src/ophyd_async/epics/adpilatus/_pilatus.py @@ -3,7 +3,7 @@ from ophyd_async.core import PathProvider from ophyd_async.core._signal import SignalR from ophyd_async.epics.adcore._core_detector import AreaDetector -from ophyd_async.epics.adcore._core_io import NDPluginBaseIO +from ophyd_async.epics.adcore._core_io import ADBaseDatasetDescriber, NDPluginBaseIO from ophyd_async.epics.adcore._core_writer import ADWriter from ophyd_async.epics.adcore._hdf_writer import ADHDFWriter @@ -28,16 +28,23 @@ def __init__( controller, driver = PilatusController.controller_and_drv( prefix + drv_suffix, name=name, readout_time=readout_time ) + writer, fileio = writer_cls.writer_and_io( + prefix, + path_provider, + lambda: name, + ADBaseDatasetDescriber(driver), + fileio_suffix=fileio_suffix, + plugins=plugins, + ) super().__init__( - prefix=prefix, driver=driver, controller=controller, - writer_cls=writer_cls, - path_provider=path_provider, + fileio=fileio, + writer=writer, plugins=plugins, name=name, - fileio_suffix=fileio_suffix, config_sigs=config_sigs, ) self.drv = driver + self.fileio = fileio diff --git a/src/ophyd_async/epics/adsimdetector/_sim.py b/src/ophyd_async/epics/adsimdetector/_sim.py index a96f8ca3e7..d6605af528 100644 --- a/src/ophyd_async/epics/adsimdetector/_sim.py +++ b/src/ophyd_async/epics/adsimdetector/_sim.py @@ -31,15 +31,23 @@ def __init__( controller, driver = SimController.controller_and_drv( prefix + drv_suffix, name=name ) + writer, fileio = writer_cls.writer_and_io( + prefix, + path_provider, + lambda: name, + adcore.ADBaseDatasetDescriber(driver), + fileio_suffix=fileio_suffix, + plugins=plugins, + ) super().__init__( - prefix=prefix, driver=driver, controller=controller, - writer_cls=writer_cls, - fileio_suffix=fileio_suffix, - path_provider=path_provider, + fileio=fileio, + writer=writer, plugins=plugins, name=name, config_sigs=config_sigs, ) + self.drv = driver + self.fileio = fileio diff --git a/src/ophyd_async/epics/advimba/_vimba.py b/src/ophyd_async/epics/advimba/_vimba.py index ae439959b5..6325c20688 100644 --- a/src/ophyd_async/epics/advimba/_vimba.py +++ b/src/ophyd_async/epics/advimba/_vimba.py @@ -2,6 +2,7 @@ from ophyd_async.core import PathProvider, SignalR from ophyd_async.epics import adcore +from ophyd_async.epics.adcore._core_io import ADBaseDatasetDescriber from ._vimba_controller import VimbaController @@ -25,17 +26,23 @@ def __init__( controller, driver = VimbaController.controller_and_drv( prefix + drv_suffix, name=name ) + writer, fileio = writer_cls.writer_and_io( + prefix, + path_provider, + lambda: name, + ADBaseDatasetDescriber(driver), + fileio_suffix=fileio_suffix, + plugins=plugins, + ) super().__init__( - prefix=prefix, driver=driver, controller=controller, - writer_cls=writer_cls, - path_provider=path_provider, + fileio=fileio, + writer=writer, plugins=plugins, name=name, - fileio_suffix=fileio_suffix, config_sigs=config_sigs, ) - self.drv = driver + self.fileio = fileio diff --git a/tests/epics/conftest.py b/tests/epics/conftest.py index f089128c6e..2e442883a1 100644 --- a/tests/epics/conftest.py +++ b/tests/epics/conftest.py @@ -33,13 +33,18 @@ def generate_ad_standard_det( prefix + "cam1:", name=name ) - test_adstandard_det = adcore.AreaDetector[controller_cls, writer_cls]( + writer, fileio = writer_cls.writer_and_io( prefix, + static_path_provider, + lambda: name, + adcore.ADBaseDatasetDescriber(driver), + ) + + test_adstandard_det = adcore.AreaDetector[controller_cls, writer_cls]( driver, controller, - writer_cls, - static_path_provider, - {}, + fileio, + writer, name=name, ) From 8494da45ec902605fd0e9bd7fa29e9d3d76a6029 Mon Sep 17 00:00:00 2001 From: jwlodek Date: Tue, 26 Nov 2024 15:00:14 -0500 Subject: [PATCH 22/60] Make sure we don't wait for capture to be done! --- src/ophyd_async/epics/adcore/_core_writer.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/ophyd_async/epics/adcore/_core_writer.py b/src/ophyd_async/epics/adcore/_core_writer.py index 7aed65429b..03aa318e97 100644 --- a/src/ophyd_async/epics/adcore/_core_writer.py +++ b/src/ophyd_async/epics/adcore/_core_writer.py @@ -121,7 +121,9 @@ async def begin_capture(self) -> None: # Overwrite num_capture to go forever await self._fileio.num_capture.set(0) # Wait for it to start, stashing the status that tells us when it finishes - self._capture_status = await set_and_wait_for_value(self._fileio.capture, True) + self._capture_status = await set_and_wait_for_value( + self._fileio.capture, True, wait_for_set_completion=False + ) async def open(self, multiplier: int = 1) -> dict[str, DataKey]: self._emitted_resource = None From 488d7eb268952e8a7f306c1e8847226e79cbb95a Mon Sep 17 00:00:00 2001 From: Jakub Wlodek Date: Mon, 9 Dec 2024 17:53:08 -0500 Subject: [PATCH 23/60] Allow for specifying whether or not to use fileio signals for dataset description --- src/ophyd_async/core/_detector.py | 1 - src/ophyd_async/epics/adaravis/_aravis.py | 8 ++++---- src/ophyd_async/epics/adcore/_core_io.py | 4 +++- src/ophyd_async/epics/adcore/_core_writer.py | 21 +++++++++++++++----- src/ophyd_async/epics/adkinetix/_kinetix.py | 6 +++--- src/ophyd_async/epics/adpilatus/_pilatus.py | 7 ++++--- src/ophyd_async/epics/adsimdetector/_sim.py | 3 ++- src/ophyd_async/epics/advimba/_vimba.py | 6 +++--- tests/epics/adaravis/test_aravis.py | 4 +++- tests/epics/adcore/test_drivers.py | 4 +++- tests/epics/adcore/test_scans.py | 4 ++-- tests/epics/adpilatus/test_pilatus.py | 1 - 12 files changed, 43 insertions(+), 26 deletions(-) diff --git a/src/ophyd_async/core/_detector.py b/src/ophyd_async/core/_detector.py index 39b3759e87..fdbc054fad 100644 --- a/src/ophyd_async/core/_detector.py +++ b/src/ophyd_async/core/_detector.py @@ -6,7 +6,6 @@ from collections.abc import AsyncGenerator, AsyncIterator, Callable, Iterator, Sequence from functools import cached_property from typing import ( - Any, Generic, TypeVar, ) diff --git a/src/ophyd_async/epics/adaravis/_aravis.py b/src/ophyd_async/epics/adaravis/_aravis.py index df3968fb39..4fa7ad548b 100644 --- a/src/ophyd_async/epics/adaravis/_aravis.py +++ b/src/ophyd_async/epics/adaravis/_aravis.py @@ -3,7 +3,6 @@ from ophyd_async.core import PathProvider from ophyd_async.core._signal import SignalR from ophyd_async.epics import adcore -from ophyd_async.epics.adcore._core_io import ADBaseDatasetDescriber from ._aravis_controller import AravisController @@ -19,13 +18,14 @@ def __init__( self, prefix: str, path_provider: PathProvider, - drv_suffix="cam1:", + drv_suffix: str = "cam1:", writer_cls: type[adcore.ADWriter] = adcore.ADHDFWriter, fileio_suffix: str | None = None, name: str = "", gpio_number: AravisController.GPIO_NUMBER = 1, config_sigs: Sequence[SignalR] = (), plugins: dict[str, adcore.NDPluginBaseIO] | None = None, + use_fileio_for_ds_describer: bool = False, ): controller, driver = AravisController.controller_and_drv( prefix + drv_suffix, gpio_number=gpio_number, name=name @@ -33,8 +33,8 @@ def __init__( writer, fileio = writer_cls.writer_and_io( prefix, path_provider, - lambda: self.name, - ADBaseDatasetDescriber(driver), + lambda: name, + ds_describer_source=driver if not use_fileio_for_ds_describer else None, fileio_suffix=fileio_suffix, plugins=plugins, ) diff --git a/src/ophyd_async/epics/adcore/_core_io.py b/src/ophyd_async/epics/adcore/_core_io.py index 40a8596cde..fb4c16e030 100644 --- a/src/ophyd_async/epics/adcore/_core_io.py +++ b/src/ophyd_async/epics/adcore/_core_io.py @@ -48,7 +48,9 @@ async def shape(self) -> tuple[int, int]: class NDPluginBaseIO(NDArrayBaseIO): def __init__(self, prefix: str, name: str = "") -> None: self.nd_array_port = epics_signal_rw_rbv(str, prefix + "NDArrayPort") - self.enable_callbacks = epics_signal_rw_rbv(Callback, prefix + "EnableCallbacks") + self.enable_callbacks = epics_signal_rw_rbv( + Callback, prefix + "EnableCallbacks" + ) self.nd_array_address = epics_signal_rw_rbv(int, prefix + "NDArrayAddress") self.array_size0 = epics_signal_r(int, prefix + "ArraySize0_RBV") self.array_size1 = epics_signal_r(int, prefix + "ArraySize1_RBV") diff --git a/src/ophyd_async/epics/adcore/_core_writer.py b/src/ophyd_async/epics/adcore/_core_writer.py index 98690f4ac0..7ef5d1e9b1 100644 --- a/src/ophyd_async/epics/adcore/_core_writer.py +++ b/src/ophyd_async/epics/adcore/_core_writer.py @@ -23,13 +23,19 @@ from ophyd_async.core._utils import DEFAULT_TIMEOUT # from ophyd_async.epics.adcore._core_logic import ADBaseDatasetDescriber -from ._core_io import ADBaseDatasetDescriber, Callback, NDFileIO, NDPluginBaseIO +from ._core_io import ( + ADBaseDatasetDescriber, + Callback, + NDArrayBaseIO, + NDFileIO, + NDPluginBaseIO, +) from ._utils import FileWriteMode -class ADWriterFormat(str, Enum): - HDF5 = ("HDF1:",) - TIFF = ("TIFF1:",) +class DatasetDescriberSource(Enum, str): + DRIVER = "driver" + FILEIO = "fileio" NDFileIOT = TypeVar("NDFileIOT", bound=NDFileIO) @@ -69,7 +75,7 @@ def writer_and_io( prefix: str, path_provider: PathProvider, name_provider: NameProvider, - dataset_describer: ADBaseDatasetDescriber, + ds_describer_source: NDArrayBaseIO | None = None, fileio_suffix: str | None = None, plugins: dict[str, NDPluginBaseIO] | None = None, ) -> tuple[ADWriterT, NDFileIOT]: @@ -85,6 +91,11 @@ def writer_and_io( fileio = fileio_cls(fileio_prefix, name=name_provider()) + if ds_describer_source is None: + dataset_describer = ADBaseDatasetDescriber(fileio) + else: + dataset_describer = ADBaseDatasetDescriber(ds_describer_source) + writer = cls( fileio, path_provider, name_provider, dataset_describer, plugins=plugins ) diff --git a/src/ophyd_async/epics/adkinetix/_kinetix.py b/src/ophyd_async/epics/adkinetix/_kinetix.py index edfbbe3b80..88467109bd 100644 --- a/src/ophyd_async/epics/adkinetix/_kinetix.py +++ b/src/ophyd_async/epics/adkinetix/_kinetix.py @@ -2,7 +2,6 @@ from ophyd_async.core import PathProvider, SignalR from ophyd_async.epics.adcore import ( - ADBaseDatasetDescriber, ADHDFWriter, ADWriter, AreaDetector, @@ -26,8 +25,9 @@ def __init__( writer_cls: type[ADWriter] = ADHDFWriter, fileio_suffix: str | None = None, name: str = "", - plugins: dict[str, NDPluginBaseIO] | None = None, config_sigs: Sequence[SignalR] = (), + plugins: dict[str, NDPluginBaseIO] | None = None, + use_fileio_for_ds_describer: bool = False, ): controller, driver = KinetixController.controller_and_drv( prefix + drv_suffix, name=name @@ -36,7 +36,7 @@ def __init__( prefix, path_provider, lambda: name, - ADBaseDatasetDescriber(driver), + ds_describer_source=driver if not use_fileio_for_ds_describer else None, fileio_suffix=fileio_suffix, plugins=plugins, ) diff --git a/src/ophyd_async/epics/adpilatus/_pilatus.py b/src/ophyd_async/epics/adpilatus/_pilatus.py index 0fc1664214..4178243d4c 100644 --- a/src/ophyd_async/epics/adpilatus/_pilatus.py +++ b/src/ophyd_async/epics/adpilatus/_pilatus.py @@ -3,7 +3,7 @@ from ophyd_async.core import PathProvider from ophyd_async.core._signal import SignalR from ophyd_async.epics.adcore._core_detector import AreaDetector -from ophyd_async.epics.adcore._core_io import ADBaseDatasetDescriber, NDPluginBaseIO +from ophyd_async.epics.adcore._core_io import NDPluginBaseIO from ophyd_async.epics.adcore._core_writer import ADWriter from ophyd_async.epics.adcore._hdf_writer import ADHDFWriter @@ -22,8 +22,9 @@ def __init__( writer_cls: type[ADWriter] = ADHDFWriter, fileio_suffix: str | None = None, name: str = "", - plugins: dict[str, NDPluginBaseIO] | None = None, config_sigs: Sequence[SignalR] = (), + plugins: dict[str, NDPluginBaseIO] | None = None, + use_fileio_for_ds_describer: bool = False, ): controller, driver = PilatusController.controller_and_drv( prefix + drv_suffix, name=name, readout_time=readout_time @@ -32,7 +33,7 @@ def __init__( prefix, path_provider, lambda: name, - ADBaseDatasetDescriber(driver), + ds_describer_source=driver if not use_fileio_for_ds_describer else None, fileio_suffix=fileio_suffix, plugins=plugins, ) diff --git a/src/ophyd_async/epics/adsimdetector/_sim.py b/src/ophyd_async/epics/adsimdetector/_sim.py index d6605af528..c2318dac7c 100644 --- a/src/ophyd_async/epics/adsimdetector/_sim.py +++ b/src/ophyd_async/epics/adsimdetector/_sim.py @@ -27,6 +27,7 @@ def __init__( name="", config_sigs: Sequence[SignalR] = (), plugins: dict[str, adcore.NDPluginBaseIO] | None = None, + use_fileio_for_ds_describer: bool = False, ): controller, driver = SimController.controller_and_drv( prefix + drv_suffix, name=name @@ -35,7 +36,7 @@ def __init__( prefix, path_provider, lambda: name, - adcore.ADBaseDatasetDescriber(driver), + ds_describer_source=driver if not use_fileio_for_ds_describer else None, fileio_suffix=fileio_suffix, plugins=plugins, ) diff --git a/src/ophyd_async/epics/advimba/_vimba.py b/src/ophyd_async/epics/advimba/_vimba.py index 6325c20688..c5e0c7b0ce 100644 --- a/src/ophyd_async/epics/advimba/_vimba.py +++ b/src/ophyd_async/epics/advimba/_vimba.py @@ -2,7 +2,6 @@ from ophyd_async.core import PathProvider, SignalR from ophyd_async.epics import adcore -from ophyd_async.epics.adcore._core_io import ADBaseDatasetDescriber from ._vimba_controller import VimbaController @@ -20,8 +19,9 @@ def __init__( writer_cls: type[adcore.ADWriter] = adcore.ADHDFWriter, fileio_suffix: str | None = None, name: str = "", - plugins: dict[str, adcore.NDPluginBaseIO] | None = None, config_sigs: Sequence[SignalR] = (), + plugins: dict[str, adcore.NDPluginBaseIO] | None = None, + use_fileio_for_ds_describer: bool = False, ): controller, driver = VimbaController.controller_and_drv( prefix + drv_suffix, name=name @@ -30,7 +30,7 @@ def __init__( prefix, path_provider, lambda: name, - ADBaseDatasetDescriber(driver), + ds_describer_source=driver if not use_fileio_for_ds_describer else None, fileio_suffix=fileio_suffix, plugins=plugins, ) diff --git a/tests/epics/adaravis/test_aravis.py b/tests/epics/adaravis/test_aravis.py index a5d8327143..5004fd6ff0 100644 --- a/tests/epics/adaravis/test_aravis.py +++ b/tests/epics/adaravis/test_aravis.py @@ -25,7 +25,9 @@ async def test_deadtime_invariant_with_exposure_time( async def test_trigger_source_set_to_gpio_line(test_adaravis: adaravis.AravisDetector): - set_mock_value(test_adaravis.drv.trigger_source, adaravis.AravisTriggerSource.FREERUN) + set_mock_value( + test_adaravis.drv.trigger_source, adaravis.AravisTriggerSource.FREERUN + ) async def trigger_and_complete(): await test_adaravis._controller.prepare( diff --git a/tests/epics/adcore/test_drivers.py b/tests/epics/adcore/test_drivers.py index 0740265c8e..00bcfa73d0 100644 --- a/tests/epics/adcore/test_drivers.py +++ b/tests/epics/adcore/test_drivers.py @@ -79,7 +79,9 @@ async def test_start_acquiring_driver_and_ensure_status_fails_after_some_time( async def wait_then_fail(): await asyncio.sleep(0.1) - set_mock_value(controller._driver.detector_state, adcore.DetectorState.DISCONNECTED) + set_mock_value( + controller._driver.detector_state, adcore.DetectorState.DISCONNECTED + ) await wait_then_fail() diff --git a/tests/epics/adcore/test_scans.py b/tests/epics/adcore/test_scans.py index 2036dd51c3..1fd54e9a98 100644 --- a/tests/epics/adcore/test_scans.py +++ b/tests/epics/adcore/test_scans.py @@ -18,7 +18,7 @@ StandardFlyer, TriggerInfo, ) -from ophyd_async.epics import adcore, adsimdetector +from ophyd_async.epics import adcore from ophyd_async.testing import set_mock_value @@ -38,7 +38,7 @@ async def stop(self): ... class DummyController(DetectorController): def __init__(self) -> None: ... async def prepare(self, trigger_info: TriggerInfo): - return AsyncStatus(asyncio.sleep(0.01)) + await AsyncStatus(asyncio.sleep(0.01)) async def arm(self): self._arm_status = AsyncStatus(asyncio.sleep(0.01)) diff --git a/tests/epics/adpilatus/test_pilatus.py b/tests/epics/adpilatus/test_pilatus.py index d39aa3daa5..2e9bd28b3c 100644 --- a/tests/epics/adpilatus/test_pilatus.py +++ b/tests/epics/adpilatus/test_pilatus.py @@ -1,6 +1,5 @@ import asyncio from collections.abc import Awaitable, Callable -from typing import cast from unittest.mock import patch import pytest From a76b70f8b169c11f1bf3252b9c6a1c04bf0ab2ec Mon Sep 17 00:00:00 2001 From: Jakub Wlodek Date: Tue, 10 Dec 2024 12:48:43 -0500 Subject: [PATCH 24/60] Revert "Allow for specifying whether or not to use fileio signals for dataset description" This reverts commit 488d7eb268952e8a7f306c1e8847226e79cbb95a. --- src/ophyd_async/core/_detector.py | 1 + src/ophyd_async/epics/adaravis/_aravis.py | 8 ++++---- src/ophyd_async/epics/adcore/_core_io.py | 4 +--- src/ophyd_async/epics/adcore/_core_writer.py | 21 +++++--------------- src/ophyd_async/epics/adkinetix/_kinetix.py | 6 +++--- src/ophyd_async/epics/adpilatus/_pilatus.py | 7 +++---- src/ophyd_async/epics/adsimdetector/_sim.py | 3 +-- src/ophyd_async/epics/advimba/_vimba.py | 6 +++--- tests/epics/adaravis/test_aravis.py | 4 +--- tests/epics/adcore/test_drivers.py | 4 +--- tests/epics/adcore/test_scans.py | 4 ++-- tests/epics/adpilatus/test_pilatus.py | 1 + 12 files changed, 26 insertions(+), 43 deletions(-) diff --git a/src/ophyd_async/core/_detector.py b/src/ophyd_async/core/_detector.py index fdbc054fad..39b3759e87 100644 --- a/src/ophyd_async/core/_detector.py +++ b/src/ophyd_async/core/_detector.py @@ -6,6 +6,7 @@ from collections.abc import AsyncGenerator, AsyncIterator, Callable, Iterator, Sequence from functools import cached_property from typing import ( + Any, Generic, TypeVar, ) diff --git a/src/ophyd_async/epics/adaravis/_aravis.py b/src/ophyd_async/epics/adaravis/_aravis.py index 4fa7ad548b..df3968fb39 100644 --- a/src/ophyd_async/epics/adaravis/_aravis.py +++ b/src/ophyd_async/epics/adaravis/_aravis.py @@ -3,6 +3,7 @@ from ophyd_async.core import PathProvider from ophyd_async.core._signal import SignalR from ophyd_async.epics import adcore +from ophyd_async.epics.adcore._core_io import ADBaseDatasetDescriber from ._aravis_controller import AravisController @@ -18,14 +19,13 @@ def __init__( self, prefix: str, path_provider: PathProvider, - drv_suffix: str = "cam1:", + drv_suffix="cam1:", writer_cls: type[adcore.ADWriter] = adcore.ADHDFWriter, fileio_suffix: str | None = None, name: str = "", gpio_number: AravisController.GPIO_NUMBER = 1, config_sigs: Sequence[SignalR] = (), plugins: dict[str, adcore.NDPluginBaseIO] | None = None, - use_fileio_for_ds_describer: bool = False, ): controller, driver = AravisController.controller_and_drv( prefix + drv_suffix, gpio_number=gpio_number, name=name @@ -33,8 +33,8 @@ def __init__( writer, fileio = writer_cls.writer_and_io( prefix, path_provider, - lambda: name, - ds_describer_source=driver if not use_fileio_for_ds_describer else None, + lambda: self.name, + ADBaseDatasetDescriber(driver), fileio_suffix=fileio_suffix, plugins=plugins, ) diff --git a/src/ophyd_async/epics/adcore/_core_io.py b/src/ophyd_async/epics/adcore/_core_io.py index fb4c16e030..40a8596cde 100644 --- a/src/ophyd_async/epics/adcore/_core_io.py +++ b/src/ophyd_async/epics/adcore/_core_io.py @@ -48,9 +48,7 @@ async def shape(self) -> tuple[int, int]: class NDPluginBaseIO(NDArrayBaseIO): def __init__(self, prefix: str, name: str = "") -> None: self.nd_array_port = epics_signal_rw_rbv(str, prefix + "NDArrayPort") - self.enable_callbacks = epics_signal_rw_rbv( - Callback, prefix + "EnableCallbacks" - ) + self.enable_callbacks = epics_signal_rw_rbv(Callback, prefix + "EnableCallbacks") self.nd_array_address = epics_signal_rw_rbv(int, prefix + "NDArrayAddress") self.array_size0 = epics_signal_r(int, prefix + "ArraySize0_RBV") self.array_size1 = epics_signal_r(int, prefix + "ArraySize1_RBV") diff --git a/src/ophyd_async/epics/adcore/_core_writer.py b/src/ophyd_async/epics/adcore/_core_writer.py index 7ef5d1e9b1..98690f4ac0 100644 --- a/src/ophyd_async/epics/adcore/_core_writer.py +++ b/src/ophyd_async/epics/adcore/_core_writer.py @@ -23,19 +23,13 @@ from ophyd_async.core._utils import DEFAULT_TIMEOUT # from ophyd_async.epics.adcore._core_logic import ADBaseDatasetDescriber -from ._core_io import ( - ADBaseDatasetDescriber, - Callback, - NDArrayBaseIO, - NDFileIO, - NDPluginBaseIO, -) +from ._core_io import ADBaseDatasetDescriber, Callback, NDFileIO, NDPluginBaseIO from ._utils import FileWriteMode -class DatasetDescriberSource(Enum, str): - DRIVER = "driver" - FILEIO = "fileio" +class ADWriterFormat(str, Enum): + HDF5 = ("HDF1:",) + TIFF = ("TIFF1:",) NDFileIOT = TypeVar("NDFileIOT", bound=NDFileIO) @@ -75,7 +69,7 @@ def writer_and_io( prefix: str, path_provider: PathProvider, name_provider: NameProvider, - ds_describer_source: NDArrayBaseIO | None = None, + dataset_describer: ADBaseDatasetDescriber, fileio_suffix: str | None = None, plugins: dict[str, NDPluginBaseIO] | None = None, ) -> tuple[ADWriterT, NDFileIOT]: @@ -91,11 +85,6 @@ def writer_and_io( fileio = fileio_cls(fileio_prefix, name=name_provider()) - if ds_describer_source is None: - dataset_describer = ADBaseDatasetDescriber(fileio) - else: - dataset_describer = ADBaseDatasetDescriber(ds_describer_source) - writer = cls( fileio, path_provider, name_provider, dataset_describer, plugins=plugins ) diff --git a/src/ophyd_async/epics/adkinetix/_kinetix.py b/src/ophyd_async/epics/adkinetix/_kinetix.py index 88467109bd..edfbbe3b80 100644 --- a/src/ophyd_async/epics/adkinetix/_kinetix.py +++ b/src/ophyd_async/epics/adkinetix/_kinetix.py @@ -2,6 +2,7 @@ from ophyd_async.core import PathProvider, SignalR from ophyd_async.epics.adcore import ( + ADBaseDatasetDescriber, ADHDFWriter, ADWriter, AreaDetector, @@ -25,9 +26,8 @@ def __init__( writer_cls: type[ADWriter] = ADHDFWriter, fileio_suffix: str | None = None, name: str = "", - config_sigs: Sequence[SignalR] = (), plugins: dict[str, NDPluginBaseIO] | None = None, - use_fileio_for_ds_describer: bool = False, + config_sigs: Sequence[SignalR] = (), ): controller, driver = KinetixController.controller_and_drv( prefix + drv_suffix, name=name @@ -36,7 +36,7 @@ def __init__( prefix, path_provider, lambda: name, - ds_describer_source=driver if not use_fileio_for_ds_describer else None, + ADBaseDatasetDescriber(driver), fileio_suffix=fileio_suffix, plugins=plugins, ) diff --git a/src/ophyd_async/epics/adpilatus/_pilatus.py b/src/ophyd_async/epics/adpilatus/_pilatus.py index 4178243d4c..0fc1664214 100644 --- a/src/ophyd_async/epics/adpilatus/_pilatus.py +++ b/src/ophyd_async/epics/adpilatus/_pilatus.py @@ -3,7 +3,7 @@ from ophyd_async.core import PathProvider from ophyd_async.core._signal import SignalR from ophyd_async.epics.adcore._core_detector import AreaDetector -from ophyd_async.epics.adcore._core_io import NDPluginBaseIO +from ophyd_async.epics.adcore._core_io import ADBaseDatasetDescriber, NDPluginBaseIO from ophyd_async.epics.adcore._core_writer import ADWriter from ophyd_async.epics.adcore._hdf_writer import ADHDFWriter @@ -22,9 +22,8 @@ def __init__( writer_cls: type[ADWriter] = ADHDFWriter, fileio_suffix: str | None = None, name: str = "", - config_sigs: Sequence[SignalR] = (), plugins: dict[str, NDPluginBaseIO] | None = None, - use_fileio_for_ds_describer: bool = False, + config_sigs: Sequence[SignalR] = (), ): controller, driver = PilatusController.controller_and_drv( prefix + drv_suffix, name=name, readout_time=readout_time @@ -33,7 +32,7 @@ def __init__( prefix, path_provider, lambda: name, - ds_describer_source=driver if not use_fileio_for_ds_describer else None, + ADBaseDatasetDescriber(driver), fileio_suffix=fileio_suffix, plugins=plugins, ) diff --git a/src/ophyd_async/epics/adsimdetector/_sim.py b/src/ophyd_async/epics/adsimdetector/_sim.py index c2318dac7c..d6605af528 100644 --- a/src/ophyd_async/epics/adsimdetector/_sim.py +++ b/src/ophyd_async/epics/adsimdetector/_sim.py @@ -27,7 +27,6 @@ def __init__( name="", config_sigs: Sequence[SignalR] = (), plugins: dict[str, adcore.NDPluginBaseIO] | None = None, - use_fileio_for_ds_describer: bool = False, ): controller, driver = SimController.controller_and_drv( prefix + drv_suffix, name=name @@ -36,7 +35,7 @@ def __init__( prefix, path_provider, lambda: name, - ds_describer_source=driver if not use_fileio_for_ds_describer else None, + adcore.ADBaseDatasetDescriber(driver), fileio_suffix=fileio_suffix, plugins=plugins, ) diff --git a/src/ophyd_async/epics/advimba/_vimba.py b/src/ophyd_async/epics/advimba/_vimba.py index c5e0c7b0ce..6325c20688 100644 --- a/src/ophyd_async/epics/advimba/_vimba.py +++ b/src/ophyd_async/epics/advimba/_vimba.py @@ -2,6 +2,7 @@ from ophyd_async.core import PathProvider, SignalR from ophyd_async.epics import adcore +from ophyd_async.epics.adcore._core_io import ADBaseDatasetDescriber from ._vimba_controller import VimbaController @@ -19,9 +20,8 @@ def __init__( writer_cls: type[adcore.ADWriter] = adcore.ADHDFWriter, fileio_suffix: str | None = None, name: str = "", - config_sigs: Sequence[SignalR] = (), plugins: dict[str, adcore.NDPluginBaseIO] | None = None, - use_fileio_for_ds_describer: bool = False, + config_sigs: Sequence[SignalR] = (), ): controller, driver = VimbaController.controller_and_drv( prefix + drv_suffix, name=name @@ -30,7 +30,7 @@ def __init__( prefix, path_provider, lambda: name, - ds_describer_source=driver if not use_fileio_for_ds_describer else None, + ADBaseDatasetDescriber(driver), fileio_suffix=fileio_suffix, plugins=plugins, ) diff --git a/tests/epics/adaravis/test_aravis.py b/tests/epics/adaravis/test_aravis.py index 5004fd6ff0..a5d8327143 100644 --- a/tests/epics/adaravis/test_aravis.py +++ b/tests/epics/adaravis/test_aravis.py @@ -25,9 +25,7 @@ async def test_deadtime_invariant_with_exposure_time( async def test_trigger_source_set_to_gpio_line(test_adaravis: adaravis.AravisDetector): - set_mock_value( - test_adaravis.drv.trigger_source, adaravis.AravisTriggerSource.FREERUN - ) + set_mock_value(test_adaravis.drv.trigger_source, adaravis.AravisTriggerSource.FREERUN) async def trigger_and_complete(): await test_adaravis._controller.prepare( diff --git a/tests/epics/adcore/test_drivers.py b/tests/epics/adcore/test_drivers.py index 00bcfa73d0..0740265c8e 100644 --- a/tests/epics/adcore/test_drivers.py +++ b/tests/epics/adcore/test_drivers.py @@ -79,9 +79,7 @@ async def test_start_acquiring_driver_and_ensure_status_fails_after_some_time( async def wait_then_fail(): await asyncio.sleep(0.1) - set_mock_value( - controller._driver.detector_state, adcore.DetectorState.DISCONNECTED - ) + set_mock_value(controller._driver.detector_state, adcore.DetectorState.DISCONNECTED) await wait_then_fail() diff --git a/tests/epics/adcore/test_scans.py b/tests/epics/adcore/test_scans.py index 1fd54e9a98..2036dd51c3 100644 --- a/tests/epics/adcore/test_scans.py +++ b/tests/epics/adcore/test_scans.py @@ -18,7 +18,7 @@ StandardFlyer, TriggerInfo, ) -from ophyd_async.epics import adcore +from ophyd_async.epics import adcore, adsimdetector from ophyd_async.testing import set_mock_value @@ -38,7 +38,7 @@ async def stop(self): ... class DummyController(DetectorController): def __init__(self) -> None: ... async def prepare(self, trigger_info: TriggerInfo): - await AsyncStatus(asyncio.sleep(0.01)) + return AsyncStatus(asyncio.sleep(0.01)) async def arm(self): self._arm_status = AsyncStatus(asyncio.sleep(0.01)) diff --git a/tests/epics/adpilatus/test_pilatus.py b/tests/epics/adpilatus/test_pilatus.py index 2e9bd28b3c..d39aa3daa5 100644 --- a/tests/epics/adpilatus/test_pilatus.py +++ b/tests/epics/adpilatus/test_pilatus.py @@ -1,5 +1,6 @@ import asyncio from collections.abc import Awaitable, Callable +from typing import cast from unittest.mock import patch import pytest From 7da935e1c8bf13dea6ecf17fa7b32ab1f96ab388 Mon Sep 17 00:00:00 2001 From: Jakub Wlodek Date: Tue, 10 Dec 2024 12:51:44 -0500 Subject: [PATCH 25/60] Fix linter errors, remove unused enum --- src/ophyd_async/core/_detector.py | 1 - src/ophyd_async/epics/adcore/_core_io.py | 4 +++- src/ophyd_async/epics/adcore/_core_writer.py | 7 ------- tests/epics/adaravis/test_aravis.py | 4 +++- tests/epics/adcore/test_drivers.py | 4 +++- tests/epics/adcore/test_scans.py | 2 +- tests/epics/adpilatus/test_pilatus.py | 1 - 7 files changed, 10 insertions(+), 13 deletions(-) diff --git a/src/ophyd_async/core/_detector.py b/src/ophyd_async/core/_detector.py index 39b3759e87..fdbc054fad 100644 --- a/src/ophyd_async/core/_detector.py +++ b/src/ophyd_async/core/_detector.py @@ -6,7 +6,6 @@ from collections.abc import AsyncGenerator, AsyncIterator, Callable, Iterator, Sequence from functools import cached_property from typing import ( - Any, Generic, TypeVar, ) diff --git a/src/ophyd_async/epics/adcore/_core_io.py b/src/ophyd_async/epics/adcore/_core_io.py index 40a8596cde..fb4c16e030 100644 --- a/src/ophyd_async/epics/adcore/_core_io.py +++ b/src/ophyd_async/epics/adcore/_core_io.py @@ -48,7 +48,9 @@ async def shape(self) -> tuple[int, int]: class NDPluginBaseIO(NDArrayBaseIO): def __init__(self, prefix: str, name: str = "") -> None: self.nd_array_port = epics_signal_rw_rbv(str, prefix + "NDArrayPort") - self.enable_callbacks = epics_signal_rw_rbv(Callback, prefix + "EnableCallbacks") + self.enable_callbacks = epics_signal_rw_rbv( + Callback, prefix + "EnableCallbacks" + ) self.nd_array_address = epics_signal_rw_rbv(int, prefix + "NDArrayAddress") self.array_size0 = epics_signal_r(int, prefix + "ArraySize0_RBV") self.array_size1 = epics_signal_r(int, prefix + "ArraySize1_RBV") diff --git a/src/ophyd_async/epics/adcore/_core_writer.py b/src/ophyd_async/epics/adcore/_core_writer.py index 98690f4ac0..bc216de412 100644 --- a/src/ophyd_async/epics/adcore/_core_writer.py +++ b/src/ophyd_async/epics/adcore/_core_writer.py @@ -1,6 +1,5 @@ import asyncio from collections.abc import AsyncGenerator, AsyncIterator -from enum import Enum from pathlib import Path from typing import Generic, TypeVar, get_args from urllib.parse import urlunparse @@ -26,12 +25,6 @@ from ._core_io import ADBaseDatasetDescriber, Callback, NDFileIO, NDPluginBaseIO from ._utils import FileWriteMode - -class ADWriterFormat(str, Enum): - HDF5 = ("HDF1:",) - TIFF = ("TIFF1:",) - - NDFileIOT = TypeVar("NDFileIOT", bound=NDFileIO) ADWriterT = TypeVar("ADWriterT", bound="ADWriter") diff --git a/tests/epics/adaravis/test_aravis.py b/tests/epics/adaravis/test_aravis.py index a5d8327143..5004fd6ff0 100644 --- a/tests/epics/adaravis/test_aravis.py +++ b/tests/epics/adaravis/test_aravis.py @@ -25,7 +25,9 @@ async def test_deadtime_invariant_with_exposure_time( async def test_trigger_source_set_to_gpio_line(test_adaravis: adaravis.AravisDetector): - set_mock_value(test_adaravis.drv.trigger_source, adaravis.AravisTriggerSource.FREERUN) + set_mock_value( + test_adaravis.drv.trigger_source, adaravis.AravisTriggerSource.FREERUN + ) async def trigger_and_complete(): await test_adaravis._controller.prepare( diff --git a/tests/epics/adcore/test_drivers.py b/tests/epics/adcore/test_drivers.py index 0740265c8e..00bcfa73d0 100644 --- a/tests/epics/adcore/test_drivers.py +++ b/tests/epics/adcore/test_drivers.py @@ -79,7 +79,9 @@ async def test_start_acquiring_driver_and_ensure_status_fails_after_some_time( async def wait_then_fail(): await asyncio.sleep(0.1) - set_mock_value(controller._driver.detector_state, adcore.DetectorState.DISCONNECTED) + set_mock_value( + controller._driver.detector_state, adcore.DetectorState.DISCONNECTED + ) await wait_then_fail() diff --git a/tests/epics/adcore/test_scans.py b/tests/epics/adcore/test_scans.py index 2036dd51c3..2d54fe596c 100644 --- a/tests/epics/adcore/test_scans.py +++ b/tests/epics/adcore/test_scans.py @@ -18,7 +18,7 @@ StandardFlyer, TriggerInfo, ) -from ophyd_async.epics import adcore, adsimdetector +from ophyd_async.epics import adcore from ophyd_async.testing import set_mock_value diff --git a/tests/epics/adpilatus/test_pilatus.py b/tests/epics/adpilatus/test_pilatus.py index d39aa3daa5..2e9bd28b3c 100644 --- a/tests/epics/adpilatus/test_pilatus.py +++ b/tests/epics/adpilatus/test_pilatus.py @@ -1,6 +1,5 @@ import asyncio from collections.abc import Awaitable, Callable -from typing import cast from unittest.mock import patch import pytest From 0cd0ddf059b76ea53e12a4ddd769c34d5ededa0e Mon Sep 17 00:00:00 2001 From: Jakub Wlodek Date: Tue, 10 Dec 2024 13:24:46 -0500 Subject: [PATCH 26/60] Change from return to await to conform to return type --- tests/epics/adcore/test_scans.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/epics/adcore/test_scans.py b/tests/epics/adcore/test_scans.py index 2d54fe596c..81dce4dc00 100644 --- a/tests/epics/adcore/test_scans.py +++ b/tests/epics/adcore/test_scans.py @@ -38,7 +38,7 @@ async def stop(self): ... class DummyController(DetectorController): def __init__(self) -> None: ... async def prepare(self, trigger_info: TriggerInfo): - return AsyncStatus(asyncio.sleep(0.01)) + await asyncio.sleep(0.01) async def arm(self): self._arm_status = AsyncStatus(asyncio.sleep(0.01)) From f1b9a4edc3838f9af182f6454cc38554198b867a Mon Sep 17 00:00:00 2001 From: Jakub Wlodek Date: Wed, 18 Dec 2024 08:50:07 -0500 Subject: [PATCH 27/60] Apply more suggestions from review --- src/ophyd_async/core/_detector.py | 4 +- src/ophyd_async/epics/adaravis/_aravis.py | 18 +++----- .../epics/adcore/_core_detector.py | 11 +++-- src/ophyd_async/epics/adcore/_core_logic.py | 24 ++-------- src/ophyd_async/epics/adcore/_core_writer.py | 28 +++++++----- src/ophyd_async/epics/adkinetix/_kinetix.py | 18 +++----- src/ophyd_async/epics/adpilatus/_pilatus.py | 19 ++++---- src/ophyd_async/epics/adsimdetector/_sim.py | 16 +++---- src/ophyd_async/epics/advimba/_vimba.py | 18 +++----- tests/epics/adaravis/test_aravis.py | 18 ++++---- tests/epics/adcore/test_drivers.py | 6 ++- tests/epics/adkinetix/test_kinetix.py | 21 ++++----- tests/epics/adpilatus/test_pilatus.py | 5 ++- tests/epics/adsimdetector/test_sim.py | 10 ++--- tests/epics/advimba/test_vimba.py | 44 ++++++++++--------- tests/epics/conftest.py | 29 +++++------- 16 files changed, 130 insertions(+), 159 deletions(-) diff --git a/src/ophyd_async/core/_detector.py b/src/ophyd_async/core/_detector.py index fdbc054fad..f596e6351e 100644 --- a/src/ophyd_async/core/_detector.py +++ b/src/ophyd_async/core/_detector.py @@ -167,8 +167,8 @@ def hints(self) -> Hints: return {} -# Add type vars for controller/type, so we can define -# StandardDetector[KinetixController, ADTIFFWriter] for example +# Add type var for controller so we can define +# StandardDetector[KinetixController, ADWriter] for example DetectorControllerT = TypeVar("DetectorControllerT", bound=DetectorController) DetectorWriterT = TypeVar("DetectorWriterT", bound=DetectorWriter) diff --git a/src/ophyd_async/epics/adaravis/_aravis.py b/src/ophyd_async/epics/adaravis/_aravis.py index df3968fb39..a912d75ea5 100644 --- a/src/ophyd_async/epics/adaravis/_aravis.py +++ b/src/ophyd_async/epics/adaravis/_aravis.py @@ -3,12 +3,12 @@ from ophyd_async.core import PathProvider from ophyd_async.core._signal import SignalR from ophyd_async.epics import adcore -from ophyd_async.epics.adcore._core_io import ADBaseDatasetDescriber from ._aravis_controller import AravisController +from ._aravis_io import AravisDriverIO -class AravisDetector(adcore.AreaDetector[AravisController, adcore.ADWriter]): +class AravisDetector(adcore.AreaDetector[AravisController]): """ Ophyd-async implementation of an ADAravis Detector. The detector may be configured for an external trigger on a GPIO port, @@ -27,14 +27,13 @@ def __init__( config_sigs: Sequence[SignalR] = (), plugins: dict[str, adcore.NDPluginBaseIO] | None = None, ): - controller, driver = AravisController.controller_and_drv( - prefix + drv_suffix, gpio_number=gpio_number, name=name - ) - writer, fileio = writer_cls.writer_and_io( + driver = AravisDriverIO(prefix + drv_suffix) + controller = AravisController(driver, gpio_number=gpio_number) + + writer = writer_cls.with_io( prefix, path_provider, - lambda: self.name, - ADBaseDatasetDescriber(driver), + dataset_source=driver, fileio_suffix=fileio_suffix, plugins=plugins, ) @@ -42,11 +41,8 @@ def __init__( super().__init__( driver=driver, controller=controller, - fileio=fileio, writer=writer, plugins=plugins, name=name, config_sigs=config_sigs, ) - self.drv = driver - self.fileio = fileio diff --git a/src/ophyd_async/epics/adcore/_core_detector.py b/src/ophyd_async/epics/adcore/_core_detector.py index 3e7d600e5e..749174eeea 100644 --- a/src/ophyd_async/epics/adcore/_core_detector.py +++ b/src/ophyd_async/epics/adcore/_core_detector.py @@ -2,24 +2,23 @@ from ophyd_async.core import SignalR, StandardDetector -from ._core_io import ADBaseIO, NDFileIO, NDPluginBaseIO +from ._core_io import ADBaseIO, NDPluginBaseIO from ._core_logic import ADBaseControllerT -from ._core_writer import ADWriterT +from ._core_writer import ADWriter -class AreaDetector(StandardDetector[ADBaseControllerT, ADWriterT]): +class AreaDetector(StandardDetector[ADBaseControllerT, ADWriter]): def __init__( self, driver: ADBaseIO, controller: ADBaseControllerT, - fileio: NDFileIO, - writer: ADWriterT, + writer: ADWriter, plugins: dict[str, NDPluginBaseIO] | None = None, config_sigs: Sequence[SignalR] = (), name: str = "", ): self.drv = driver - self.fileio = fileio + self.fileio = writer._fileio # noqa: SLF001 if plugins is not None: for name, plugin in plugins.items(): diff --git a/src/ophyd_async/epics/adcore/_core_logic.py b/src/ophyd_async/epics/adcore/_core_logic.py index 5b4da2476f..29bbb3f53d 100644 --- a/src/ophyd_async/epics/adcore/_core_logic.py +++ b/src/ophyd_async/epics/adcore/_core_logic.py @@ -1,5 +1,5 @@ import asyncio -from typing import Any, Generic, TypeVar, get_args +from typing import Generic, TypeVar from ophyd_async.core import ( DEFAULT_TIMEOUT, @@ -34,26 +34,10 @@ def __init__( self.frame_timeout = DEFAULT_TIMEOUT self._arm_status: AsyncStatus | None = None - @classmethod - def controller_and_drv( - cls: type[ADBaseControllerT], - prefix: str, - good_states: frozenset[DetectorState] = DEFAULT_GOOD_STATES, - name: str = "", - ) -> tuple[ADBaseControllerT, ADBaseIOT]: - try: - driver_cls = get_args(cls.__orig_bases__[0])[0] # type: ignore - except IndexError as err: - raise RuntimeError("Driver IO class for controller not specified!") from err - - driver = driver_cls(prefix, name=name) - controller = cls(driver, good_states=good_states) - return controller, driver - def get_deadtime(self, exposure: float | None) -> float: - return 0.002 + return 0.001 - async def prepare(self, trigger_info: TriggerInfo) -> Any: + async def prepare(self, trigger_info: TriggerInfo) -> None: assert ( trigger_info.trigger == DetectorTrigger.INTERNAL ), "fly scanning (i.e. external triggering) is not supported for this device" @@ -119,7 +103,7 @@ async def start_acquiring_driver_and_ensure_status(self) -> AsyncStatus: status = await set_and_wait_for_value( self._driver.acquire, True, - timeout=self.frame_timeout, + timeout=DEFAULT_TIMEOUT, wait_for_set_completion=False, ) diff --git a/src/ophyd_async/epics/adcore/_core_writer.py b/src/ophyd_async/epics/adcore/_core_writer.py index bc216de412..fa188175b2 100644 --- a/src/ophyd_async/epics/adcore/_core_writer.py +++ b/src/ophyd_async/epics/adcore/_core_writer.py @@ -22,7 +22,13 @@ from ophyd_async.core._utils import DEFAULT_TIMEOUT # from ophyd_async.epics.adcore._core_logic import ADBaseDatasetDescriber -from ._core_io import ADBaseDatasetDescriber, Callback, NDFileIO, NDPluginBaseIO +from ._core_io import ( + ADBaseDatasetDescriber, + Callback, + NDArrayBaseIO, + NDFileIO, + NDPluginBaseIO, +) from ._utils import FileWriteMode NDFileIOT = TypeVar("NDFileIOT", bound=NDFileIO) @@ -57,31 +63,31 @@ def __init__( self._filename_template = "%s%s_%6.6d" @classmethod - def writer_and_io( + def with_io( cls: type[ADWriterT], prefix: str, path_provider: PathProvider, - name_provider: NameProvider, - dataset_describer: ADBaseDatasetDescriber, + dataset_source: NDArrayBaseIO | None = None, fileio_suffix: str | None = None, plugins: dict[str, NDPluginBaseIO] | None = None, - ) -> tuple[ADWriterT, NDFileIOT]: + ) -> ADWriterT: try: fileio_cls = get_args(cls.__orig_bases__[0])[0] # type: ignore except IndexError as err: raise RuntimeError("File IO class for writer not specified!") from err - if fileio_suffix is None: - fileio_prefix = prefix + cls.default_suffix - else: - fileio_prefix = prefix + fileio_suffix + fileio = fileio_cls(prefix + (fileio_suffix or cls.default_suffix)) + dataset_describer = ADBaseDatasetDescriber(dataset_source or fileio) - fileio = fileio_cls(fileio_prefix, name=name_provider()) + def name_provider() -> str: + if fileio.parent == "Not attached to a detector": + raise RuntimeError("Initializing writer without parent detector!") + return fileio.parent.name writer = cls( fileio, path_provider, name_provider, dataset_describer, plugins=plugins ) - return writer, fileio + return writer async def begin_capture(self) -> None: info = self._path_provider(device_name=self._name_provider()) diff --git a/src/ophyd_async/epics/adkinetix/_kinetix.py b/src/ophyd_async/epics/adkinetix/_kinetix.py index edfbbe3b80..95fb432ed3 100644 --- a/src/ophyd_async/epics/adkinetix/_kinetix.py +++ b/src/ophyd_async/epics/adkinetix/_kinetix.py @@ -2,7 +2,6 @@ from ophyd_async.core import PathProvider, SignalR from ophyd_async.epics.adcore import ( - ADBaseDatasetDescriber, ADHDFWriter, ADWriter, AreaDetector, @@ -10,9 +9,10 @@ ) from ._kinetix_controller import KinetixController +from ._kinetix_io import KinetixDriverIO -class KinetixDetector(AreaDetector[KinetixController, ADWriter]): +class KinetixDetector(AreaDetector[KinetixController]): """ Ophyd-async implementation of an ADKinetix Detector. https://github.com/NSLS-II/ADKinetix @@ -29,14 +29,13 @@ def __init__( plugins: dict[str, NDPluginBaseIO] | None = None, config_sigs: Sequence[SignalR] = (), ): - controller, driver = KinetixController.controller_and_drv( - prefix + drv_suffix, name=name - ) - writer, fileio = writer_cls.writer_and_io( + driver = KinetixDriverIO(prefix + drv_suffix) + controller = KinetixController(driver) + + writer = writer_cls.with_io( prefix, path_provider, - lambda: name, - ADBaseDatasetDescriber(driver), + dataset_source=driver, fileio_suffix=fileio_suffix, plugins=plugins, ) @@ -44,11 +43,8 @@ def __init__( super().__init__( driver=driver, controller=controller, - fileio=fileio, writer=writer, plugins=plugins, name=name, config_sigs=config_sigs, ) - self.drv = driver - self.fileio = fileio diff --git a/src/ophyd_async/epics/adpilatus/_pilatus.py b/src/ophyd_async/epics/adpilatus/_pilatus.py index 0fc1664214..ee2961bd95 100644 --- a/src/ophyd_async/epics/adpilatus/_pilatus.py +++ b/src/ophyd_async/epics/adpilatus/_pilatus.py @@ -3,14 +3,15 @@ from ophyd_async.core import PathProvider from ophyd_async.core._signal import SignalR from ophyd_async.epics.adcore._core_detector import AreaDetector -from ophyd_async.epics.adcore._core_io import ADBaseDatasetDescriber, NDPluginBaseIO +from ophyd_async.epics.adcore._core_io import NDPluginBaseIO from ophyd_async.epics.adcore._core_writer import ADWriter from ophyd_async.epics.adcore._hdf_writer import ADHDFWriter from ._pilatus_controller import PilatusController, PilatusReadoutTime +from ._pilatus_io import PilatusDriverIO -class PilatusDetector(AreaDetector[PilatusController, ADWriter]): +class PilatusDetector(AreaDetector[PilatusController]): """A Pilatus StandardDetector writing HDF files""" def __init__( @@ -25,14 +26,13 @@ def __init__( plugins: dict[str, NDPluginBaseIO] | None = None, config_sigs: Sequence[SignalR] = (), ): - controller, driver = PilatusController.controller_and_drv( - prefix + drv_suffix, name=name, readout_time=readout_time - ) - writer, fileio = writer_cls.writer_and_io( + driver = PilatusDriverIO(prefix + drv_suffix) + controller = PilatusController(driver) + + writer = writer_cls.with_io( prefix, path_provider, - lambda: name, - ADBaseDatasetDescriber(driver), + dataset_source=driver, fileio_suffix=fileio_suffix, plugins=plugins, ) @@ -40,11 +40,8 @@ def __init__( super().__init__( driver=driver, controller=controller, - fileio=fileio, writer=writer, plugins=plugins, name=name, config_sigs=config_sigs, ) - self.drv = driver - self.fileio = fileio diff --git a/src/ophyd_async/epics/adsimdetector/_sim.py b/src/ophyd_async/epics/adsimdetector/_sim.py index d6605af528..c399137672 100644 --- a/src/ophyd_async/epics/adsimdetector/_sim.py +++ b/src/ophyd_async/epics/adsimdetector/_sim.py @@ -16,7 +16,7 @@ def __init__( super().__init__(driver, good_states=good_states) -class SimDetector(adcore.AreaDetector[SimController, adcore.ADWriter]): +class SimDetector(adcore.AreaDetector[SimController]): def __init__( self, prefix: str, @@ -28,14 +28,13 @@ def __init__( config_sigs: Sequence[SignalR] = (), plugins: dict[str, adcore.NDPluginBaseIO] | None = None, ): - controller, driver = SimController.controller_and_drv( - prefix + drv_suffix, name=name - ) - writer, fileio = writer_cls.writer_and_io( + driver = SimDriverIO(prefix + drv_suffix) + controller = SimController(driver) + + writer = writer_cls.with_io( prefix, path_provider, - lambda: name, - adcore.ADBaseDatasetDescriber(driver), + dataset_source=driver, fileio_suffix=fileio_suffix, plugins=plugins, ) @@ -43,11 +42,8 @@ def __init__( super().__init__( driver=driver, controller=controller, - fileio=fileio, writer=writer, plugins=plugins, name=name, config_sigs=config_sigs, ) - self.drv = driver - self.fileio = fileio diff --git a/src/ophyd_async/epics/advimba/_vimba.py b/src/ophyd_async/epics/advimba/_vimba.py index 6325c20688..85b4b19dfe 100644 --- a/src/ophyd_async/epics/advimba/_vimba.py +++ b/src/ophyd_async/epics/advimba/_vimba.py @@ -2,12 +2,12 @@ from ophyd_async.core import PathProvider, SignalR from ophyd_async.epics import adcore -from ophyd_async.epics.adcore._core_io import ADBaseDatasetDescriber from ._vimba_controller import VimbaController +from ._vimba_io import VimbaDriverIO -class VimbaDetector(adcore.AreaDetector[VimbaController, adcore.ADWriter]): +class VimbaDetector(adcore.AreaDetector[VimbaController]): """ Ophyd-async implementation of an ADVimba Detector. """ @@ -23,14 +23,13 @@ def __init__( plugins: dict[str, adcore.NDPluginBaseIO] | None = None, config_sigs: Sequence[SignalR] = (), ): - controller, driver = VimbaController.controller_and_drv( - prefix + drv_suffix, name=name - ) - writer, fileio = writer_cls.writer_and_io( + driver = VimbaDriverIO(prefix + drv_suffix) + controller = VimbaController(driver) + + writer = writer_cls.with_io( prefix, path_provider, - lambda: name, - ADBaseDatasetDescriber(driver), + dataset_source=driver, fileio_suffix=fileio_suffix, plugins=plugins, ) @@ -38,11 +37,8 @@ def __init__( super().__init__( driver=driver, controller=controller, - fileio=fileio, writer=writer, plugins=plugins, name=name, config_sigs=config_sigs, ) - self.drv = driver - self.fileio = fileio diff --git a/tests/epics/adaravis/test_aravis.py b/tests/epics/adaravis/test_aravis.py index 5004fd6ff0..9333d84588 100644 --- a/tests/epics/adaravis/test_aravis.py +++ b/tests/epics/adaravis/test_aravis.py @@ -1,4 +1,5 @@ import re +from typing import cast import pytest @@ -13,7 +14,7 @@ @pytest.fixture def test_adaravis(ad_standard_det_factory) -> adaravis.AravisDetector: - return ad_standard_det_factory(adaravis.AravisController) + return ad_standard_det_factory(adaravis.AravisDetector) @pytest.mark.parametrize("exposure_time", [0.0, 0.1, 1.0, 10.0, 100.0]) @@ -25,9 +26,8 @@ async def test_deadtime_invariant_with_exposure_time( async def test_trigger_source_set_to_gpio_line(test_adaravis: adaravis.AravisDetector): - set_mock_value( - test_adaravis.drv.trigger_source, adaravis.AravisTriggerSource.FREERUN - ) + driver = cast(adaravis.AravisDriverIO, test_adaravis.drv) + set_mock_value(driver.trigger_source, adaravis.AravisTriggerSource.FREERUN) async def trigger_and_complete(): await test_adaravis._controller.prepare( @@ -40,23 +40,23 @@ async def trigger_and_complete(): ) ) # Prevent timeouts - set_mock_value(test_adaravis.drv.acquire, True) + set_mock_value(driver.acquire, True) # Default TriggerSource - assert (await test_adaravis.drv.trigger_source.get_value()) == "Freerun" + assert (await driver.trigger_source.get_value()) == "Freerun" test_adaravis._controller.set_external_trigger_gpio(1) # TriggerSource not changed by setting gpio - assert (await test_adaravis.drv.trigger_source.get_value()) == "Freerun" + assert (await driver.trigger_source.get_value()) == "Freerun" await trigger_and_complete() # TriggerSource changes - assert (await test_adaravis.drv.trigger_source.get_value()) == "Line1" + assert (await driver.trigger_source.get_value()) == "Line1" test_adaravis._controller.set_external_trigger_gpio(3) # TriggerSource not changed by setting gpio await trigger_and_complete() - assert (await test_adaravis.drv.trigger_source.get_value()) == "Line3" + assert (await driver.trigger_source.get_value()) == "Line3" def test_gpio_pin_limited(test_adaravis: adaravis.AravisDetector): diff --git a/tests/epics/adcore/test_drivers.py b/tests/epics/adcore/test_drivers.py index 00bcfa73d0..2d65c6470b 100644 --- a/tests/epics/adcore/test_drivers.py +++ b/tests/epics/adcore/test_drivers.py @@ -1,4 +1,5 @@ import asyncio +from unittest.mock import patch import pytest @@ -67,6 +68,7 @@ async def test_start_acquiring_driver_and_ensure_status_flags_immediate_failure( await acquiring +@patch("ophyd_async.core._detector.DEFAULT_TIMEOUT", 0.2) async def test_start_acquiring_driver_and_ensure_status_fails_after_some_time( controller: adcore.ADBaseController, ): @@ -78,13 +80,15 @@ async def test_start_acquiring_driver_and_ensure_status_fails_after_some_time( set_mock_value(controller._driver.detector_state, adcore.DetectorState.IDLE) async def wait_then_fail(): - await asyncio.sleep(0.1) + await asyncio.sleep(0) set_mock_value( controller._driver.detector_state, adcore.DetectorState.DISCONNECTED ) await wait_then_fail() + controller.frame_timeout = 0.1 + acquiring = await controller.start_acquiring_driver_and_ensure_status() with pytest.raises( diff --git a/tests/epics/adkinetix/test_kinetix.py b/tests/epics/adkinetix/test_kinetix.py index 0cac5759b8..27bedf3394 100644 --- a/tests/epics/adkinetix/test_kinetix.py +++ b/tests/epics/adkinetix/test_kinetix.py @@ -1,3 +1,5 @@ +from typing import cast + import pytest from ophyd_async.core import ( @@ -11,7 +13,7 @@ @pytest.fixture def test_adkinetix(ad_standard_det_factory) -> adkinetix.KinetixDetector: - return ad_standard_det_factory(adkinetix.KinetixController) + return ad_standard_det_factory(adkinetix.KinetixDetector) async def test_get_deadtime( @@ -22,9 +24,8 @@ async def test_get_deadtime( async def test_trigger_modes(test_adkinetix: adkinetix.KinetixDetector): - set_mock_value( - test_adkinetix.drv.trigger_mode, adkinetix.KinetixTriggerMode.INTERNAL - ) + driver = cast(adkinetix.KinetixDriverIO, test_adkinetix.drv) + set_mock_value(driver.trigger_mode, adkinetix.KinetixTriggerMode.INTERNAL) async def setup_trigger_mode(trig_mode: DetectorTrigger): await test_adkinetix._controller.prepare( @@ -33,22 +34,22 @@ async def setup_trigger_mode(trig_mode: DetectorTrigger): await test_adkinetix._controller.arm() await test_adkinetix._controller.wait_for_idle() # Prevent timeouts - set_mock_value(test_adkinetix.drv.acquire, True) + set_mock_value(driver.acquire, True) # Default TriggerSource - assert (await test_adkinetix.drv.trigger_mode.get_value()) == "Internal" + assert (await driver.trigger_mode.get_value()) == "Internal" await setup_trigger_mode(DetectorTrigger.EDGE_TRIGGER) - assert (await test_adkinetix.drv.trigger_mode.get_value()) == "Rising Edge" + assert (await driver.trigger_mode.get_value()) == "Rising Edge" await setup_trigger_mode(DetectorTrigger.CONSTANT_GATE) - assert (await test_adkinetix.drv.trigger_mode.get_value()) == "Exp. Gate" + assert (await driver.trigger_mode.get_value()) == "Exp. Gate" await setup_trigger_mode(DetectorTrigger.INTERNAL) - assert (await test_adkinetix.drv.trigger_mode.get_value()) == "Internal" + assert (await driver.trigger_mode.get_value()) == "Internal" await setup_trigger_mode(DetectorTrigger.VARIABLE_GATE) - assert (await test_adkinetix.drv.trigger_mode.get_value()) == "Exp. Gate" + assert (await driver.trigger_mode.get_value()) == "Exp. Gate" async def test_hints_from_hdf_writer(test_adkinetix: adkinetix.KinetixDetector): diff --git a/tests/epics/adpilatus/test_pilatus.py b/tests/epics/adpilatus/test_pilatus.py index 2e9bd28b3c..e715b3e5fe 100644 --- a/tests/epics/adpilatus/test_pilatus.py +++ b/tests/epics/adpilatus/test_pilatus.py @@ -1,5 +1,6 @@ import asyncio from collections.abc import Awaitable, Callable +from typing import cast from unittest.mock import patch import pytest @@ -14,7 +15,7 @@ @pytest.fixture def test_adpilatus(ad_standard_det_factory) -> adpilatus.PilatusDetector: - return ad_standard_det_factory(adpilatus.PilatusController) + return ad_standard_det_factory(adpilatus.PilatusDetector) async def test_deadtime_overridable(test_adpilatus: adpilatus.PilatusDetector): @@ -139,7 +140,7 @@ async def dummy_open(multiplier: int = 0): async def test_pilatus_controller(test_adpilatus: adpilatus.PilatusDetector): pilatus = test_adpilatus._controller - pilatus_driver = test_adpilatus.drv + pilatus_driver = cast(adpilatus.PilatusDriverIO, test_adpilatus.drv) set_mock_value(pilatus_driver.armed, True) await pilatus.prepare( TriggerInfo(number_of_triggers=1, trigger=DetectorTrigger.CONSTANT_GATE) diff --git a/tests/epics/adsimdetector/test_sim.py b/tests/epics/adsimdetector/test_sim.py index fa749a307e..7cbcdd6d95 100644 --- a/tests/epics/adsimdetector/test_sim.py +++ b/tests/epics/adsimdetector/test_sim.py @@ -26,22 +26,22 @@ @pytest.fixture def test_adsimdetector(ad_standard_det_factory: Callable) -> adsimdetector.SimDetector: - return ad_standard_det_factory(adsimdetector.SimController) + return ad_standard_det_factory(adsimdetector.SimDetector) @pytest.fixture def test_adsimdetector_tiff( ad_standard_det_factory: Callable, ) -> adsimdetector.SimDetector: - return ad_standard_det_factory(adsimdetector.SimController, adcore.ADTIFFWriter) + return ad_standard_det_factory(adsimdetector.SimDetector, adcore.ADTIFFWriter) @pytest.fixture def two_test_adsimdetectors( ad_standard_det_factory: Callable, ) -> Sequence[adsimdetector.SimDetector]: - deta = ad_standard_det_factory(adsimdetector.SimController) - detb = ad_standard_det_factory(adsimdetector.SimController, number=2) + deta = ad_standard_det_factory(adsimdetector.SimDetector) + detb = ad_standard_det_factory(adsimdetector.SimDetector, number=2) return deta, detb @@ -239,7 +239,7 @@ async def test_detector_writes_to_file( tmp_path: Path, ): test_adsimdetector: adsimdetector.SimDetector = ad_standard_det_factory( - adsimdetector.SimController, writer_cls + adsimdetector.SimDetector, writer_cls ) names = [] diff --git a/tests/epics/advimba/test_vimba.py b/tests/epics/advimba/test_vimba.py index c8f101396b..10c8d8091f 100644 --- a/tests/epics/advimba/test_vimba.py +++ b/tests/epics/advimba/test_vimba.py @@ -1,3 +1,5 @@ +from typing import cast + import pytest from ophyd_async.core import ( @@ -16,7 +18,7 @@ @pytest.fixture def test_advimba(ad_standard_det_factory) -> advimba.VimbaDetector: - return ad_standard_det_factory(advimba.VimbaController, adcore.ADHDFWriter) + return ad_standard_det_factory(advimba.VimbaDetector, adcore.ADHDFWriter) async def test_get_deadtime( @@ -27,9 +29,11 @@ async def test_get_deadtime( async def test_arming_trig_modes(test_advimba: advimba.VimbaDetector): - set_mock_value(test_advimba.drv.trigger_source, VimbaTriggerSource.FREERUN) - set_mock_value(test_advimba.drv.trigger_mode, VimbaOnOff.OFF) - set_mock_value(test_advimba.drv.exposure_mode, VimbaExposeOutMode.TIMED) + driver = cast(advimba.VimbaDriverIO, test_advimba.drv) + + set_mock_value(driver.trigger_source, VimbaTriggerSource.FREERUN) + set_mock_value(driver.trigger_mode, VimbaOnOff.OFF) + set_mock_value(driver.exposure_mode, VimbaExposeOutMode.TIMED) async def setup_trigger_mode(trig_mode: DetectorTrigger): await test_advimba._controller.prepare( @@ -38,32 +42,32 @@ async def setup_trigger_mode(trig_mode: DetectorTrigger): await test_advimba._controller.arm() await test_advimba._controller.wait_for_idle() # Prevent timeouts - set_mock_value(test_advimba.drv.acquire, True) + set_mock_value(driver.acquire, True) # Default TriggerSource - assert (await test_advimba.drv.trigger_source.get_value()) == "Freerun" - assert (await test_advimba.drv.trigger_mode.get_value()) == "Off" - assert (await test_advimba.drv.exposure_mode.get_value()) == "Timed" + assert (await driver.trigger_source.get_value()) == "Freerun" + assert (await driver.trigger_mode.get_value()) == "Off" + assert (await driver.exposure_mode.get_value()) == "Timed" await setup_trigger_mode(DetectorTrigger.EDGE_TRIGGER) - assert (await test_advimba.drv.trigger_source.get_value()) == "Line1" - assert (await test_advimba.drv.trigger_mode.get_value()) == "On" - assert (await test_advimba.drv.exposure_mode.get_value()) == "Timed" + assert (await driver.trigger_source.get_value()) == "Line1" + assert (await driver.trigger_mode.get_value()) == "On" + assert (await driver.exposure_mode.get_value()) == "Timed" await setup_trigger_mode(DetectorTrigger.CONSTANT_GATE) - assert (await test_advimba.drv.trigger_source.get_value()) == "Line1" - assert (await test_advimba.drv.trigger_mode.get_value()) == "On" - assert (await test_advimba.drv.exposure_mode.get_value()) == "TriggerWidth" + assert (await driver.trigger_source.get_value()) == "Line1" + assert (await driver.trigger_mode.get_value()) == "On" + assert (await driver.exposure_mode.get_value()) == "TriggerWidth" await setup_trigger_mode(DetectorTrigger.INTERNAL) - assert (await test_advimba.drv.trigger_source.get_value()) == "Freerun" - assert (await test_advimba.drv.trigger_mode.get_value()) == "Off" - assert (await test_advimba.drv.exposure_mode.get_value()) == "Timed" + assert (await driver.trigger_source.get_value()) == "Freerun" + assert (await driver.trigger_mode.get_value()) == "Off" + assert (await driver.exposure_mode.get_value()) == "Timed" await setup_trigger_mode(DetectorTrigger.VARIABLE_GATE) - assert (await test_advimba.drv.trigger_source.get_value()) == "Line1" - assert (await test_advimba.drv.trigger_mode.get_value()) == "On" - assert (await test_advimba.drv.exposure_mode.get_value()) == "TriggerWidth" + assert (await driver.trigger_source.get_value()) == "Line1" + assert (await driver.trigger_mode.get_value()) == "On" + assert (await driver.exposure_mode.get_value()) == "TriggerWidth" async def test_hints_from_hdf_writer(test_advimba: advimba.VimbaDetector): diff --git a/tests/epics/conftest.py b/tests/epics/conftest.py index ecbb683e1f..08141c7d59 100644 --- a/tests/epics/conftest.py +++ b/tests/epics/conftest.py @@ -14,38 +14,29 @@ def ad_standard_det_factory( RE: RunEngine, static_path_provider, ) -> Callable[ - [type[adcore.ADBaseController], type[adcore.ADWriter], int], adcore.AreaDetector + [type[adcore.AreaDetector], type[adcore.ADWriter], int], adcore.AreaDetector ]: def generate_ad_standard_det( - controller_cls: type[adcore.ADBaseController], + detector_cls: type[adcore.AreaDetector], writer_cls: type[adcore.ADWriter] = adcore.ADHDFWriter, number=1, + **kwargs, ) -> adcore.AreaDetector: # Dynamically generate a name based on the class of controller - detector_name = controller_cls.__name__ - if detector_name.endswith("Controller"): - detector_name = detector_name[: -len("Controller")] + detector_name = detector_cls.__name__ + if detector_name.endswith("Detector"): + detector_name = detector_name[: -len("Detector")] with DeviceCollector(mock=True): prefix = f"{detector_name.upper()}{number}:" name = f"test_ad{detector_name.lower()}{number}" - controller, driver = controller_cls.controller_and_drv( - prefix + "cam1:", name=name - ) - - writer, fileio = writer_cls.writer_and_io( + test_adstandard_det = detector_cls( prefix, static_path_provider, - lambda: name, - adcore.ADBaseDatasetDescriber(driver), - ) - - test_adstandard_det = adcore.AreaDetector[controller_cls, writer_cls]( - driver, - controller, - fileio, - writer, + # The inner areaDetector class wants the instantaiated object, + # but the outward facing classes want a writer class type + writer_cls=writer_cls, # type: ignore name=name, ) From d84f2b46cf7ab0412251237a1fbf7b58c187eaf2 Mon Sep 17 00:00:00 2001 From: Jakub Wlodek Date: Wed, 18 Dec 2024 09:29:56 -0500 Subject: [PATCH 28/60] Replace instances of DeviceCollector to init_devices --- tests/epics/adcore/test_single_trigger.py | 4 ++-- tests/epics/adcore/test_writers.py | 2 +- tests/epics/conftest.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/epics/adcore/test_single_trigger.py b/tests/epics/adcore/test_single_trigger.py index 5c0dcf70e5..080a6d3338 100644 --- a/tests/epics/adcore/test_single_trigger.py +++ b/tests/epics/adcore/test_single_trigger.py @@ -4,13 +4,13 @@ from bluesky.run_engine import RunEngine import ophyd_async.plan_stubs as ops -from ophyd_async.core import DeviceCollector +from ophyd_async.core import init_devices from ophyd_async.epics import adcore @pytest.fixture async def single_trigger_det_with_stats(): - async with DeviceCollector(mock=True): + async with init_devices(mock=True): stats = adcore.NDPluginStatsIO("PREFIX:STATS", name="stats") det = adcore.SingleTriggerDetector( drv=adcore.ADBaseIO("PREFIX:DRV"), diff --git a/tests/epics/adcore/test_writers.py b/tests/epics/adcore/test_writers.py index c4ad2f8931..c5922ee23d 100644 --- a/tests/epics/adcore/test_writers.py +++ b/tests/epics/adcore/test_writers.py @@ -45,7 +45,7 @@ async def hdf_writer( async def tiff_writer( RE, static_path_provider: StaticPathProvider ) -> adcore.ADTIFFWriter: - async with DeviceCollector(mock=True): + async with init_devices(mock=True): tiff = adcore.NDFileIO("TIFF:") return adcore.ADTIFFWriter( diff --git a/tests/epics/conftest.py b/tests/epics/conftest.py index 6669d40898..9b846553e0 100644 --- a/tests/epics/conftest.py +++ b/tests/epics/conftest.py @@ -30,7 +30,7 @@ def generate_ad_standard_det( with init_devices(mock=True): prefix = f"{detector_name.upper()}{number}:" name = f"test_ad{detector_name.lower()}{number}" - + test_adstandard_det = detector_cls( prefix, static_path_provider, From a5a42c6d2e7add5746c9d399a131a678082ac86f Mon Sep 17 00:00:00 2001 From: thomashopkins32 Date: Thu, 19 Dec 2024 10:06:52 -0500 Subject: [PATCH 29/60] Started moving multiplier; 'multiplier' name change in progress --- src/ophyd_async/core/_detector.py | 22 +++++++++++--------- src/ophyd_async/core/_hdf_dataset.py | 4 ++-- src/ophyd_async/epics/adcore/_core_writer.py | 13 ++++++------ src/ophyd_async/epics/adcore/_hdf_writer.py | 11 +++++----- 4 files changed, 26 insertions(+), 24 deletions(-) diff --git a/src/ophyd_async/core/_detector.py b/src/ophyd_async/core/_detector.py index f596e6351e..96b2c3bfcb 100644 --- a/src/ophyd_async/core/_detector.py +++ b/src/ophyd_async/core/_detector.py @@ -65,11 +65,11 @@ class TriggerInfo(BaseModel): livetime: float | None = Field(default=None, ge=0) #: What is the maximum timeout on waiting for a frame frame_timeout: float | None = Field(default=None, gt=0) - #: How many triggers make up a single StreamDatum index, to allow multiple frames - #: from a faster detector to be zipped with a single frame from a slow detector - #: e.g. if num=10 and multiplier=5 then the detector will take 10 frames, - #: but publish 2 indices, and describe() will show a shape of (5, h, w) - multiplier: int = 1 + #: The number of triggers are grouped into a single StreamDatum index. + #: A batch_size > 1 can be useful to have frames from a faster detector able to be zipped with a single frame from a slower detector. + #: E.g. if number_of_triggers=10 and batch_size=5 then the detector will take 10 frames, + #: but publish 2 StreamDatum indices, and describe() will show a shape of (5, h, w) for each. + batch_size: NonNegativeInt = 1 @computed_field @cached_property @@ -107,7 +107,7 @@ async def prepare(self, trigger_info: TriggerInfo) -> None: exposure time. deadtime Defaults to None. This is the minimum deadtime between triggers. - multiplier The number of triggers grouped into a single StreamDatum + batch_size The number of triggers grouped into a single StreamDatum index. """ @@ -133,12 +133,14 @@ class DetectorWriter(ABC): (e.g. an HDF5 file)""" @abstractmethod - async def open(self, multiplier: int = 1) -> dict[str, DataKey]: + async def open(self, batch_size: int = 1) -> dict[str, DataKey]: """Open writer and wait for it to be ready for data. Args: - multiplier: Each StreamDatum index corresponds to this many - written exposures + batch_size: The number of triggers are grouped into a single StreamDatum index. + A batch_size > 1 can be useful to have frames from a faster detector able to be zipped with a single frame from a slow detector. + E.g. if number_of_triggers=10 and batch_size=5 then the detector will take 10 frames, + but publish 2 StreamDatum indices, and describe() will show a shape of (5, h, w) for each. Returns: Output for ``describe()`` @@ -329,7 +331,7 @@ async def prepare(self, value: TriggerInfo) -> None: ) self._initial_frame = await self._writer.get_indices_written() self._describe, _ = await asyncio.gather( - self._writer.open(value.multiplier), self._controller.prepare(value) + self._writer.open(value.batch_size), self._controller.prepare(value) ) if value.trigger != DetectorTrigger.INTERNAL: await self._controller.arm() diff --git a/src/ophyd_async/core/_hdf_dataset.py b/src/ophyd_async/core/_hdf_dataset.py index 79cb9c432a..03289b2579 100644 --- a/src/ophyd_async/core/_hdf_dataset.py +++ b/src/ophyd_async/core/_hdf_dataset.py @@ -18,7 +18,7 @@ class HDFDataset: dataset: str shape: Sequence[int] = field(default_factory=tuple) dtype_numpy: str = "" - multiplier: int = 1 + frequency_ratio: int = 1 swmr: bool = False # Represents explicit chunk size written to disk. chunk_shape: tuple[int, ...] = () @@ -67,7 +67,7 @@ def __init__( parameters={ "dataset": ds.dataset, "swmr": ds.swmr, - "multiplier": ds.multiplier, + "group_factor": ds.group_factor, "chunk_shape": ds.chunk_shape, }, uid=None, diff --git a/src/ophyd_async/epics/adcore/_core_writer.py b/src/ophyd_async/epics/adcore/_core_writer.py index fa188175b2..864935f76c 100644 --- a/src/ophyd_async/epics/adcore/_core_writer.py +++ b/src/ophyd_async/epics/adcore/_core_writer.py @@ -59,7 +59,7 @@ def __init__( self._emitted_resource = None self._capture_status: AsyncStatus | None = None - self._multiplier = 1 + self._batch_size = 1 self._filename_template = "%s%s_%6.6d" @classmethod @@ -123,10 +123,10 @@ async def begin_capture(self) -> None: self._fileio.capture, True, wait_for_set_completion=False ) - async def open(self, multiplier: int = 1) -> dict[str, DataKey]: + async def open(self, batch_size: int = 1) -> dict[str, DataKey]: self._emitted_resource = None self._last_emitted = 0 - self._multiplier = multiplier + self._batch_size = batch_size frame_shape = await self._dataset_describer.shape() dtype_numpy = await self._dataset_describer.np_datatype() @@ -135,7 +135,7 @@ async def open(self, multiplier: int = 1) -> dict[str, DataKey]: describe = { self._name_provider(): DataKey( source=self._name_provider(), - shape=list(frame_shape), + shape=list((batch_size,) + frame_shape), dtype="array", dtype_numpy=dtype_numpy, external="STREAM:", @@ -148,11 +148,11 @@ async def observe_indices_written( ) -> AsyncGenerator[int, None]: """Wait until a specific index is ready to be collected""" async for num_captured in observe_value(self._fileio.num_captured, timeout): - yield num_captured // self._multiplier + yield num_captured // self._batch_size async def get_indices_written(self) -> int: num_captured = await self._fileio.num_captured.get_value() - return num_captured // self._multiplier + return num_captured // self._batch_size async def collect_stream_docs( self, indices_written: int @@ -183,6 +183,7 @@ async def collect_stream_docs( uri=uri, data_key=self._name_provider(), parameters={ + # TODO: Is the following assumption accurate or should this be `self._batch_size`? # Assume that we always write 1 frame per file/chunk "chunk_shape": (1, *frame_shape), # Include file template for reconstruction in consolidator diff --git a/src/ophyd_async/epics/adcore/_hdf_writer.py b/src/ophyd_async/epics/adcore/_hdf_writer.py index c9c2e94268..92420227dd 100644 --- a/src/ophyd_async/epics/adcore/_hdf_writer.py +++ b/src/ophyd_async/epics/adcore/_hdf_writer.py @@ -48,7 +48,7 @@ def __init__( self._file: HDFFile | None = None self._include_file_number = False - async def open(self, multiplier: int = 1) -> dict[str, DataKey]: + async def open(self, frequency_ratio: int = 1) -> dict[str, DataKey]: self._file = None # Setting HDF writer specific signals @@ -74,8 +74,7 @@ async def open(self, multiplier: int = 1) -> dict[str, DataKey]: name = self._name_provider() detector_shape = await self._dataset_describer.shape() np_dtype = await self._dataset_describer.np_datatype() - self._multiplier = multiplier - outer_shape = (multiplier,) if multiplier > 1 else () + self._frequency_ratio = frequency_ratio # Determine number of frames that will be saved per HDF chunk frames_per_chunk = await self._fileio.num_frames_chunks.get_value() @@ -87,7 +86,7 @@ async def open(self, multiplier: int = 1) -> dict[str, DataKey]: dataset="/entry/data/data", shape=detector_shape, dtype_numpy=np_dtype, - multiplier=multiplier, + frequency_ratio=frequency_ratio, chunk_shape=(frames_per_chunk, *detector_shape), ) ] @@ -115,7 +114,7 @@ async def open(self, multiplier: int = 1) -> dict[str, DataKey]: f"/entry/instrument/NDAttributes/{datakey}", (), np_datatype, - multiplier, + frequency_ratio, # NDAttributes appear to always be configured with # this chunk size chunk_shape=(16384,), @@ -125,7 +124,7 @@ async def open(self, multiplier: int = 1) -> dict[str, DataKey]: describe = { ds.data_key: DataKey( source=self._fileio.full_file_name.source, - shape=list(outer_shape + tuple(ds.shape)), + shape=list((frequency_ratio,) + tuple(ds.shape)), dtype="array" if ds.shape else "number", dtype_numpy=ds.dtype_numpy, external="STREAM:", From 05dd89c89b7e6e609f20770c4ce6da91672be8d2 Mon Sep 17 00:00:00 2001 From: Jakub Wlodek Date: Fri, 20 Dec 2024 11:38:31 -0500 Subject: [PATCH 30/60] Update src/ophyd_async/epics/adaravis/_aravis_controller.py Co-authored-by: Tom C (DLS) <101418278+coretl@users.noreply.github.com> --- src/ophyd_async/epics/adaravis/_aravis_controller.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/src/ophyd_async/epics/adaravis/_aravis_controller.py b/src/ophyd_async/epics/adaravis/_aravis_controller.py index 172b3c58bc..f43601c4a4 100644 --- a/src/ophyd_async/epics/adaravis/_aravis_controller.py +++ b/src/ophyd_async/epics/adaravis/_aravis_controller.py @@ -29,18 +29,6 @@ def __init__( super().__init__(driver, good_states=good_states) self.gpio_number = gpio_number - @classmethod - def controller_and_drv( - cls: type[AravisControllerT], - prefix: str, - good_states: frozenset[adcore.DetectorState] = adcore.DEFAULT_GOOD_STATES, - name: str = "", - gpio_number: GPIO_NUMBER = 1, - ) -> tuple[AravisControllerT, AravisDriverIO]: - driver_cls = get_args(cls.__orig_bases__[0])[0] # type: ignore - driver = driver_cls(prefix, name=name) - controller = cls(driver, good_states=good_states, gpio_number=gpio_number) - return controller, driver def get_deadtime(self, exposure: float | None) -> float: return _HIGHEST_POSSIBLE_DEADTIME From 3fcd541700025af68956a6b6f84610af49c1a718 Mon Sep 17 00:00:00 2001 From: Jakub Wlodek Date: Fri, 20 Dec 2024 11:38:45 -0500 Subject: [PATCH 31/60] Update src/ophyd_async/epics/adaravis/_aravis_controller.py Co-authored-by: Tom C (DLS) <101418278+coretl@users.noreply.github.com> --- src/ophyd_async/epics/adaravis/_aravis_controller.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/src/ophyd_async/epics/adaravis/_aravis_controller.py b/src/ophyd_async/epics/adaravis/_aravis_controller.py index f43601c4a4..806d2bcf68 100644 --- a/src/ophyd_async/epics/adaravis/_aravis_controller.py +++ b/src/ophyd_async/epics/adaravis/_aravis_controller.py @@ -70,15 +70,3 @@ def _get_trigger_info( else: return (AravisTriggerMode.ON, f"Line{self.gpio_number}") # type: ignore - def get_external_trigger_gpio(self): - return self.gpio_number - - def set_external_trigger_gpio(self, gpio_number: GPIO_NUMBER): - supported_gpio_numbers = get_args(AravisController.GPIO_NUMBER) - if gpio_number not in supported_gpio_numbers: - raise ValueError( - f"{self.__class__.__name__} only supports the following GPIO " - f"indices: {supported_gpio_numbers} but was asked to " - f"use {gpio_number}" - ) - self.gpio_number = gpio_number From 87d5fddec073ab59a12974203c92e862147cccc2 Mon Sep 17 00:00:00 2001 From: Jakub Wlodek Date: Fri, 20 Dec 2024 11:39:08 -0500 Subject: [PATCH 32/60] Update src/ophyd_async/epics/adcore/_core_writer.py Co-authored-by: Tom C (DLS) <101418278+coretl@users.noreply.github.com> --- src/ophyd_async/epics/adcore/_core_writer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ophyd_async/epics/adcore/_core_writer.py b/src/ophyd_async/epics/adcore/_core_writer.py index fa188175b2..ca6069117d 100644 --- a/src/ophyd_async/epics/adcore/_core_writer.py +++ b/src/ophyd_async/epics/adcore/_core_writer.py @@ -49,7 +49,7 @@ def __init__( plugins: dict[str, NDPluginBaseIO] | None = None, ) -> None: self._plugins = plugins - self._fileio = fileio + self.fileio = fileio self._path_provider = path_provider self._name_provider = name_provider self._dataset_describer = dataset_describer From 52d712ede6d242dad004ad14e8b0f362fbd8c5ac Mon Sep 17 00:00:00 2001 From: Jakub Wlodek Date: Fri, 20 Dec 2024 11:39:22 -0500 Subject: [PATCH 33/60] Update src/ophyd_async/epics/adcore/_core_logic.py Co-authored-by: Tom C (DLS) <101418278+coretl@users.noreply.github.com> --- src/ophyd_async/epics/adcore/_core_logic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ophyd_async/epics/adcore/_core_logic.py b/src/ophyd_async/epics/adcore/_core_logic.py index 29bbb3f53d..c1af0365d1 100644 --- a/src/ophyd_async/epics/adcore/_core_logic.py +++ b/src/ophyd_async/epics/adcore/_core_logic.py @@ -29,7 +29,7 @@ def __init__( driver: ADBaseIOT, good_states: frozenset[DetectorState] = DEFAULT_GOOD_STATES, ) -> None: - self._driver = driver + self.driver = driver self.good_states = good_states self.frame_timeout = DEFAULT_TIMEOUT self._arm_status: AsyncStatus | None = None From e46cbd4b355a32da140195f010d20d621fba182f Mon Sep 17 00:00:00 2001 From: Jakub Wlodek Date: Fri, 20 Dec 2024 11:39:33 -0500 Subject: [PATCH 34/60] Update src/ophyd_async/epics/adcore/_core_detector.py Co-authored-by: Tom C (DLS) <101418278+coretl@users.noreply.github.com> --- src/ophyd_async/epics/adcore/_core_detector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ophyd_async/epics/adcore/_core_detector.py b/src/ophyd_async/epics/adcore/_core_detector.py index 749174eeea..59be7a2caa 100644 --- a/src/ophyd_async/epics/adcore/_core_detector.py +++ b/src/ophyd_async/epics/adcore/_core_detector.py @@ -18,7 +18,7 @@ def __init__( name: str = "", ): self.drv = driver - self.fileio = writer._fileio # noqa: SLF001 + self.fileio = writer.fileio if plugins is not None: for name, plugin in plugins.items(): From fbb895e85f2d8399c396a2988e8b3e5d451604a7 Mon Sep 17 00:00:00 2001 From: Jakub Wlodek Date: Fri, 20 Dec 2024 11:40:09 -0500 Subject: [PATCH 35/60] Update src/ophyd_async/epics/adcore/_core_detector.py Co-authored-by: Tom C (DLS) <101418278+coretl@users.noreply.github.com> --- src/ophyd_async/epics/adcore/_core_detector.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/ophyd_async/epics/adcore/_core_detector.py b/src/ophyd_async/epics/adcore/_core_detector.py index 59be7a2caa..e4c31abb9c 100644 --- a/src/ophyd_async/epics/adcore/_core_detector.py +++ b/src/ophyd_async/epics/adcore/_core_detector.py @@ -10,14 +10,13 @@ class AreaDetector(StandardDetector[ADBaseControllerT, ADWriter]): def __init__( self, - driver: ADBaseIO, controller: ADBaseControllerT, writer: ADWriter, plugins: dict[str, NDPluginBaseIO] | None = None, config_sigs: Sequence[SignalR] = (), name: str = "", ): - self.drv = driver + self.driver = controller.driver self.fileio = writer.fileio if plugins is not None: From 6cd79a6eb041441d42c32292ab16b3c197af6ef1 Mon Sep 17 00:00:00 2001 From: Jakub Wlodek Date: Fri, 20 Dec 2024 11:40:32 -0500 Subject: [PATCH 36/60] Update src/ophyd_async/epics/adcore/_core_writer.py Co-authored-by: Tom C (DLS) <101418278+coretl@users.noreply.github.com> --- src/ophyd_async/epics/adcore/_core_writer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ophyd_async/epics/adcore/_core_writer.py b/src/ophyd_async/epics/adcore/_core_writer.py index ca6069117d..9a04390590 100644 --- a/src/ophyd_async/epics/adcore/_core_writer.py +++ b/src/ophyd_async/epics/adcore/_core_writer.py @@ -48,7 +48,7 @@ def __init__( mimetype: str = "", plugins: dict[str, NDPluginBaseIO] | None = None, ) -> None: - self._plugins = plugins + self._plugins = plugins or {} self.fileio = fileio self._path_provider = path_provider self._name_provider = name_provider From 05e6fbdefcce638b97bdf370ad20419934eea1e0 Mon Sep 17 00:00:00 2001 From: Jakub Wlodek Date: Mon, 23 Dec 2024 11:00:35 -0500 Subject: [PATCH 37/60] Fix all tests aside from 3.12 unawaited coro after applying suggestions --- src/ophyd_async/epics/adaravis/_aravis.py | 1 - .../epics/adaravis/_aravis_controller.py | 14 ++-- .../epics/adcore/_core_detector.py | 4 +- src/ophyd_async/epics/adcore/_core_logic.py | 19 ++--- src/ophyd_async/epics/adcore/_core_writer.py | 34 ++++---- src/ophyd_async/epics/adcore/_hdf_writer.py | 77 +++++++++---------- src/ophyd_async/epics/adcore/_tiff_writer.py | 2 +- src/ophyd_async/epics/adkinetix/_kinetix.py | 1 - .../epics/adkinetix/_kinetix_controller.py | 8 +- src/ophyd_async/epics/adpilatus/_pilatus.py | 1 - .../epics/adpilatus/_pilatus_controller.py | 8 +- src/ophyd_async/epics/adsimdetector/_sim.py | 4 +- src/ophyd_async/epics/advimba/_vimba.py | 1 - .../epics/advimba/_vimba_controller.py | 14 ++-- tests/core/test_protocol.py | 2 +- tests/epics/adaravis/test_aravis.py | 21 +---- tests/epics/adcore/test_drivers.py | 24 +++--- tests/epics/adcore/test_scans.py | 12 +-- tests/epics/adcore/test_writers.py | 4 +- tests/epics/adkinetix/test_kinetix.py | 2 +- tests/epics/adpilatus/test_pilatus.py | 12 +-- tests/epics/adsimdetector/test_sim.py | 42 +++++----- tests/epics/advimba/test_vimba.py | 2 +- tests/epics/conftest.py | 18 ++--- 24 files changed, 150 insertions(+), 177 deletions(-) diff --git a/src/ophyd_async/epics/adaravis/_aravis.py b/src/ophyd_async/epics/adaravis/_aravis.py index a912d75ea5..abfc8a5859 100644 --- a/src/ophyd_async/epics/adaravis/_aravis.py +++ b/src/ophyd_async/epics/adaravis/_aravis.py @@ -39,7 +39,6 @@ def __init__( ) super().__init__( - driver=driver, controller=controller, writer=writer, plugins=plugins, diff --git a/src/ophyd_async/epics/adaravis/_aravis_controller.py b/src/ophyd_async/epics/adaravis/_aravis_controller.py index 806d2bcf68..f847d52a66 100644 --- a/src/ophyd_async/epics/adaravis/_aravis_controller.py +++ b/src/ophyd_async/epics/adaravis/_aravis_controller.py @@ -1,5 +1,5 @@ import asyncio -from typing import Literal, TypeVar, get_args +from typing import Literal, TypeVar from ophyd_async.core import ( DetectorTrigger, @@ -29,7 +29,6 @@ def __init__( super().__init__(driver, good_states=good_states) self.gpio_number = gpio_number - def get_deadtime(self, exposure: float | None) -> float: return _HIGHEST_POSSIBLE_DEADTIME @@ -39,16 +38,16 @@ async def prepare(self, trigger_info: TriggerInfo): else: image_mode = adcore.ImageMode.MULTIPLE if (exposure := trigger_info.livetime) is not None: - await self._driver.acquire_time.set(exposure) + await self.driver.acquire_time.set(exposure) trigger_mode, trigger_source = self._get_trigger_info(trigger_info.trigger) # trigger mode must be set first and on it's own! - await self._driver.trigger_mode.set(trigger_mode) + await self.driver.trigger_mode.set(trigger_mode) await asyncio.gather( - self._driver.trigger_source.set(trigger_source), - self._driver.num_images.set(trigger_info.total_number_of_triggers), - self._driver.image_mode.set(image_mode), + self.driver.trigger_source.set(trigger_source), + self.driver.num_images.set(trigger_info.total_number_of_triggers), + self.driver.image_mode.set(image_mode), ) def _get_trigger_info( @@ -69,4 +68,3 @@ def _get_trigger_info( return AravisTriggerMode.OFF, AravisTriggerSource.FREERUN else: return (AravisTriggerMode.ON, f"Line{self.gpio_number}") # type: ignore - diff --git a/src/ophyd_async/epics/adcore/_core_detector.py b/src/ophyd_async/epics/adcore/_core_detector.py index e4c31abb9c..3224f25476 100644 --- a/src/ophyd_async/epics/adcore/_core_detector.py +++ b/src/ophyd_async/epics/adcore/_core_detector.py @@ -2,7 +2,7 @@ from ophyd_async.core import SignalR, StandardDetector -from ._core_io import ADBaseIO, NDPluginBaseIO +from ._core_io import NDPluginBaseIO from ._core_logic import ADBaseControllerT from ._core_writer import ADWriter @@ -26,7 +26,7 @@ def __init__( super().__init__( controller, writer, - (self.drv.acquire_period, self.drv.acquire_time, *config_sigs), + (self.driver.acquire_period, self.driver.acquire_time, *config_sigs), name=name, ) diff --git a/src/ophyd_async/epics/adcore/_core_logic.py b/src/ophyd_async/epics/adcore/_core_logic.py index c1af0365d1..a0da204374 100644 --- a/src/ophyd_async/epics/adcore/_core_logic.py +++ b/src/ophyd_async/epics/adcore/_core_logic.py @@ -34,19 +34,16 @@ def __init__( self.frame_timeout = DEFAULT_TIMEOUT self._arm_status: AsyncStatus | None = None - def get_deadtime(self, exposure: float | None) -> float: - return 0.001 - async def prepare(self, trigger_info: TriggerInfo) -> None: assert ( trigger_info.trigger == DetectorTrigger.INTERNAL ), "fly scanning (i.e. external triggering) is not supported for this device" self.frame_timeout = ( - DEFAULT_TIMEOUT + await self._driver.acquire_time.get_value() + DEFAULT_TIMEOUT + await self.driver.acquire_time.get_value() ) await asyncio.gather( - self._driver.num_images.set(trigger_info.total_number_of_triggers), - self._driver.image_mode.set(ImageMode.MULTIPLE), + self.driver.num_images.set(trigger_info.total_number_of_triggers), + self.driver.image_mode.set(ImageMode.MULTIPLE), ) async def arm(self): @@ -59,7 +56,7 @@ async def wait_for_idle(self): async def disarm(self): # We can't use caput callback as we already used it in arm() and we can't have # 2 or they will deadlock - await stop_busy_record(self._driver.acquire, False, timeout=1) + await stop_busy_record(self.driver.acquire, False, timeout=1) async def set_exposure_time_and_acquire_period_if_supplied( self, @@ -81,8 +78,8 @@ async def set_exposure_time_and_acquire_period_if_supplied( if exposure is not None: full_frame_time = exposure + self.get_deadtime(exposure) await asyncio.gather( - self._driver.acquire_time.set(exposure, timeout=timeout), - self._driver.acquire_period.set(full_frame_time, timeout=timeout), + self.driver.acquire_time.set(exposure, timeout=timeout), + self.driver.acquire_period.set(full_frame_time, timeout=timeout), ) async def start_acquiring_driver_and_ensure_status(self) -> AsyncStatus: @@ -101,7 +98,7 @@ async def start_acquiring_driver_and_ensure_status(self) -> AsyncStatus: """ status = await set_and_wait_for_value( - self._driver.acquire, + self.driver.acquire, True, timeout=DEFAULT_TIMEOUT, wait_for_set_completion=False, @@ -111,7 +108,7 @@ async def complete_acquisition() -> None: """NOTE: possible race condition here between the callback from set_and_wait_for_value and the detector state updating.""" await status - state = await self._driver.detector_state.get_value() + state = await self.driver.detector_state.get_value() if state not in self.good_states: raise ValueError( f"Final detector state {state.value} not " diff --git a/src/ophyd_async/epics/adcore/_core_writer.py b/src/ophyd_async/epics/adcore/_core_writer.py index 9a04390590..4102778b6f 100644 --- a/src/ophyd_async/epics/adcore/_core_writer.py +++ b/src/ophyd_async/epics/adcore/_core_writer.py @@ -92,35 +92,35 @@ def name_provider() -> str: async def begin_capture(self) -> None: info = self._path_provider(device_name=self._name_provider()) - await self._fileio.enable_callbacks.set(Callback.ENABLE) + await self.fileio.enable_callbacks.set(Callback.ENABLE) # Set the directory creation depth first, since dir creation callback happens # when directory path PV is processed. - await self._fileio.create_directory.set(info.create_dir_depth) + await self.fileio.create_directory.set(info.create_dir_depth) await asyncio.gather( # See https://github.com/bluesky/ophyd-async/issues/122 - self._fileio.file_path.set(str(info.directory_path)), - self._fileio.file_name.set(info.filename), - self._fileio.file_write_mode.set(FileWriteMode.STREAM), + self.fileio.file_path.set(str(info.directory_path)), + self.fileio.file_name.set(info.filename), + self.fileio.file_write_mode.set(FileWriteMode.STREAM), # For non-HDF file writers, use AD file templating mechanism # for generating multi-image datasets - self._fileio.file_template.set( + self.fileio.file_template.set( self._filename_template + self._file_extension ), - self._fileio.auto_increment.set(True), - self._fileio.file_number.set(0), + self.fileio.auto_increment.set(True), + self.fileio.file_number.set(0), ) assert ( - await self._fileio.file_path_exists.get_value() + await self.fileio.file_path_exists.get_value() ), f"File path {info.directory_path} for file plugin does not exist!" # Overwrite num_capture to go forever - await self._fileio.num_capture.set(0) + await self.fileio.num_capture.set(0) # Wait for it to start, stashing the status that tells us when it finishes self._capture_status = await set_and_wait_for_value( - self._fileio.capture, True, wait_for_set_completion=False + self.fileio.capture, True, wait_for_set_completion=False ) async def open(self, multiplier: int = 1) -> dict[str, DataKey]: @@ -147,11 +147,11 @@ async def observe_indices_written( self, timeout=DEFAULT_TIMEOUT ) -> AsyncGenerator[int, None]: """Wait until a specific index is ready to be collected""" - async for num_captured in observe_value(self._fileio.num_captured, timeout): + async for num_captured in observe_value(self.fileio.num_captured, timeout): yield num_captured // self._multiplier async def get_indices_written(self) -> int: - num_captured = await self._fileio.num_captured.get_value() + num_captured = await self.fileio.num_captured.get_value() return num_captured // self._multiplier async def collect_stream_docs( @@ -159,8 +159,8 @@ async def collect_stream_docs( ) -> AsyncIterator[StreamAsset]: if indices_written: if not self._emitted_resource: - file_path = Path(await self._fileio.file_path.get_value()) - file_name = await self._fileio.file_name.get_value() + file_path = Path(await self.fileio.file_path.get_value()) + file_name = await self.fileio.file_name.get_value() file_template = file_name + "_{:06d}" + self._file_extension frame_shape = await self._dataset_describer.shape() @@ -208,8 +208,8 @@ async def collect_stream_docs( async def close(self): # Already done a caput callback in _capture_status, so can't do one here - await self._fileio.capture.set(False, wait=False) - await wait_for_value(self._fileio.capture, False, DEFAULT_TIMEOUT) + await self.fileio.capture.set(False, wait=False) + await wait_for_value(self.fileio.capture, False, DEFAULT_TIMEOUT) if self._capture_status: # We kicked off an open, so wait for it to return await self._capture_status diff --git a/src/ophyd_async/epics/adcore/_hdf_writer.py b/src/ophyd_async/epics/adcore/_hdf_writer.py index c9c2e94268..48fb7e5724 100644 --- a/src/ophyd_async/epics/adcore/_hdf_writer.py +++ b/src/ophyd_async/epics/adcore/_hdf_writer.py @@ -54,13 +54,13 @@ async def open(self, multiplier: int = 1) -> dict[str, DataKey]: # Setting HDF writer specific signals # Make sure we are using chunk auto-sizing - await asyncio.gather(self._fileio.chunk_size_auto.set(True)) + await asyncio.gather(self.fileio.chunk_size_auto.set(True)) await asyncio.gather( - self._fileio.num_extra_dims.set(0), - self._fileio.lazy_open.set(True), - self._fileio.swmr_mode.set(True), - self._fileio.xml_file_name.set(""), + self.fileio.num_extra_dims.set(0), + self.fileio.lazy_open.set(True), + self.fileio.swmr_mode.set(True), + self.fileio.xml_file_name.set(""), ) # By default, don't add file number to filename @@ -78,7 +78,7 @@ async def open(self, multiplier: int = 1) -> dict[str, DataKey]: outer_shape = (multiplier,) if multiplier > 1 else () # Determine number of frames that will be saved per HDF chunk - frames_per_chunk = await self._fileio.num_frames_chunks.get_value() + frames_per_chunk = await self.fileio.num_frames_chunks.get_value() # Add the main data self._datasets = [ @@ -92,39 +92,38 @@ async def open(self, multiplier: int = 1) -> dict[str, DataKey]: ) ] # And all the scalar datasets - if self._plugins is not None: - for plugin in self._plugins.values(): - maybe_xml = await plugin.nd_attributes_file.get_value() - # This is the check that ADCore does to see if it is an XML string - # rather than a filename to parse - if "" in maybe_xml: - root = ET.fromstring(maybe_xml) - for child in root: - datakey = child.attrib["name"] - if child.attrib.get("type", "EPICS_PV") == "EPICS_PV": - np_datatype = convert_pv_dtype_to_np( - child.attrib.get("dbrtype", "DBR_NATIVE") - ) - else: - np_datatype = convert_param_dtype_to_np( - child.attrib.get("datatype", "INT") - ) - self._datasets.append( - HDFDataset( - datakey, - f"/entry/instrument/NDAttributes/{datakey}", - (), - np_datatype, - multiplier, - # NDAttributes appear to always be configured with - # this chunk size - chunk_shape=(16384,), - ) + for plugin in self._plugins.values(): + maybe_xml = await plugin.nd_attributes_file.get_value() + # This is the check that ADCore does to see if it is an XML string + # rather than a filename to parse + if "" in maybe_xml: + root = ET.fromstring(maybe_xml) + for child in root: + datakey = child.attrib["name"] + if child.attrib.get("type", "EPICS_PV") == "EPICS_PV": + np_datatype = convert_pv_dtype_to_np( + child.attrib.get("dbrtype", "DBR_NATIVE") ) + else: + np_datatype = convert_param_dtype_to_np( + child.attrib.get("datatype", "INT") + ) + self._datasets.append( + HDFDataset( + datakey, + f"/entry/instrument/NDAttributes/{datakey}", + (), + np_datatype, + multiplier, + # NDAttributes appear to always be configured with + # this chunk size + chunk_shape=(16384,), + ) + ) describe = { ds.data_key: DataKey( - source=self._fileio.full_file_name.source, + source=self.fileio.full_file_name.source, shape=list(outer_shape + tuple(ds.shape)), dtype="array" if ds.shape else "number", dtype_numpy=ds.dtype_numpy, @@ -138,10 +137,10 @@ async def collect_stream_docs( self, indices_written: int ) -> AsyncIterator[StreamAsset]: # TODO: fail if we get dropped frames - await self._fileio.flush_now.set(True) + await self.fileio.flush_now.set(True) if indices_written: if not self._file: - path = Path(await self._fileio.full_file_name.get_value()) + path = Path(await self.fileio.full_file_name.get_value()) self._file = HDFFile( # See https://github.com/bluesky/ophyd-async/issues/122 path, @@ -158,8 +157,8 @@ async def collect_stream_docs( async def close(self): # Already done a caput callback in _capture_status, so can't do one here - await self._fileio.capture.set(False, wait=False) - await wait_for_value(self._fileio.capture, False, DEFAULT_TIMEOUT) + await self.fileio.capture.set(False, wait=False) + await wait_for_value(self.fileio.capture, False, DEFAULT_TIMEOUT) if self._capture_status: # We kicked off an open, so wait for it to return await self._capture_status diff --git a/src/ophyd_async/epics/adcore/_tiff_writer.py b/src/ophyd_async/epics/adcore/_tiff_writer.py index 8ccb681c95..21ed0829ac 100644 --- a/src/ophyd_async/epics/adcore/_tiff_writer.py +++ b/src/ophyd_async/epics/adcore/_tiff_writer.py @@ -24,4 +24,4 @@ def __init__( file_extension=".tiff", mimetype="multipart/related;type=image/tiff", ) - self.tiff = self._fileio + self.tiff = self.fileio diff --git a/src/ophyd_async/epics/adkinetix/_kinetix.py b/src/ophyd_async/epics/adkinetix/_kinetix.py index 95fb432ed3..a40da42298 100644 --- a/src/ophyd_async/epics/adkinetix/_kinetix.py +++ b/src/ophyd_async/epics/adkinetix/_kinetix.py @@ -41,7 +41,6 @@ def __init__( ) super().__init__( - driver=driver, controller=controller, writer=writer, plugins=plugins, diff --git a/src/ophyd_async/epics/adkinetix/_kinetix_controller.py b/src/ophyd_async/epics/adkinetix/_kinetix_controller.py index 635508118a..91ade01dc2 100644 --- a/src/ophyd_async/epics/adkinetix/_kinetix_controller.py +++ b/src/ophyd_async/epics/adkinetix/_kinetix_controller.py @@ -29,14 +29,14 @@ def get_deadtime(self, exposure: float | None) -> float: async def prepare(self, trigger_info: TriggerInfo): await asyncio.gather( - self._driver.trigger_mode.set( + self.driver.trigger_mode.set( KINETIX_TRIGGER_MODE_MAP[trigger_info.trigger] ), - self._driver.num_images.set(trigger_info.total_number_of_triggers), - self._driver.image_mode.set(adcore.ImageMode.MULTIPLE), + self.driver.num_images.set(trigger_info.total_number_of_triggers), + self.driver.image_mode.set(adcore.ImageMode.MULTIPLE), ) if trigger_info.livetime is not None and trigger_info.trigger not in [ DetectorTrigger.VARIABLE_GATE, DetectorTrigger.CONSTANT_GATE, ]: - await self._driver.acquire_time.set(trigger_info.livetime) + await self.driver.acquire_time.set(trigger_info.livetime) diff --git a/src/ophyd_async/epics/adpilatus/_pilatus.py b/src/ophyd_async/epics/adpilatus/_pilatus.py index ee2961bd95..800f4caaf0 100644 --- a/src/ophyd_async/epics/adpilatus/_pilatus.py +++ b/src/ophyd_async/epics/adpilatus/_pilatus.py @@ -38,7 +38,6 @@ def __init__( ) super().__init__( - driver=driver, controller=controller, writer=writer, plugins=plugins, diff --git a/src/ophyd_async/epics/adpilatus/_pilatus_controller.py b/src/ophyd_async/epics/adpilatus/_pilatus_controller.py index d478fc565c..87bda8fd63 100644 --- a/src/ophyd_async/epics/adpilatus/_pilatus_controller.py +++ b/src/ophyd_async/epics/adpilatus/_pilatus_controller.py @@ -68,13 +68,13 @@ async def prepare(self, trigger_info: TriggerInfo): trigger_info.livetime ) await asyncio.gather( - self._driver.trigger_mode.set(self._get_trigger_mode(trigger_info.trigger)), - self._driver.num_images.set( + self.driver.trigger_mode.set(self._get_trigger_mode(trigger_info.trigger)), + self.driver.num_images.set( 999_999 if trigger_info.total_number_of_triggers == 0 else trigger_info.total_number_of_triggers ), - self._driver.image_mode.set(adcore.ImageMode.MULTIPLE), + self.driver.image_mode.set(adcore.ImageMode.MULTIPLE), ) async def arm(self): @@ -85,7 +85,7 @@ async def arm(self): # is actually ready. Should wait for that too or we risk dropping # a frame await wait_for_value( - self._driver.armed, + self.driver.armed, True, timeout=DEFAULT_TIMEOUT, ) diff --git a/src/ophyd_async/epics/adsimdetector/_sim.py b/src/ophyd_async/epics/adsimdetector/_sim.py index c399137672..b4b354da5c 100644 --- a/src/ophyd_async/epics/adsimdetector/_sim.py +++ b/src/ophyd_async/epics/adsimdetector/_sim.py @@ -15,6 +15,9 @@ def __init__( ) -> None: super().__init__(driver, good_states=good_states) + def get_deadtime(self, exposure: float | None) -> float: + return 0.001 + class SimDetector(adcore.AreaDetector[SimController]): def __init__( @@ -40,7 +43,6 @@ def __init__( ) super().__init__( - driver=driver, controller=controller, writer=writer, plugins=plugins, diff --git a/src/ophyd_async/epics/advimba/_vimba.py b/src/ophyd_async/epics/advimba/_vimba.py index 85b4b19dfe..2d04ad938b 100644 --- a/src/ophyd_async/epics/advimba/_vimba.py +++ b/src/ophyd_async/epics/advimba/_vimba.py @@ -35,7 +35,6 @@ def __init__( ) super().__init__( - driver=driver, controller=controller, writer=writer, plugins=plugins, diff --git a/src/ophyd_async/epics/advimba/_vimba_controller.py b/src/ophyd_async/epics/advimba/_vimba_controller.py index 53307123a9..065b633aa8 100644 --- a/src/ophyd_async/epics/advimba/_vimba_controller.py +++ b/src/ophyd_async/epics/advimba/_vimba_controller.py @@ -36,17 +36,17 @@ def get_deadtime(self, exposure: float | None) -> float: async def prepare(self, trigger_info: TriggerInfo): await asyncio.gather( - self._driver.trigger_mode.set(TRIGGER_MODE[trigger_info.trigger]), - self._driver.exposure_mode.set(EXPOSE_OUT_MODE[trigger_info.trigger]), - self._driver.num_images.set(trigger_info.total_number_of_triggers), - self._driver.image_mode.set(adcore.ImageMode.MULTIPLE), + self.driver.trigger_mode.set(TRIGGER_MODE[trigger_info.trigger]), + self.driver.exposure_mode.set(EXPOSE_OUT_MODE[trigger_info.trigger]), + self.driver.num_images.set(trigger_info.total_number_of_triggers), + self.driver.image_mode.set(adcore.ImageMode.MULTIPLE), ) if trigger_info.livetime is not None and trigger_info.trigger not in [ DetectorTrigger.VARIABLE_GATE, DetectorTrigger.CONSTANT_GATE, ]: - await self._driver.acquire_time.set(trigger_info.livetime) + await self.driver.acquire_time.set(trigger_info.livetime) if trigger_info.trigger != DetectorTrigger.INTERNAL: - self._driver.trigger_source.set(VimbaTriggerSource.LINE1) + self.driver.trigger_source.set(VimbaTriggerSource.LINE1) else: - self._driver.trigger_source.set(VimbaTriggerSource.FREERUN) + self.driver.trigger_source.set(VimbaTriggerSource.FREERUN) diff --git a/tests/core/test_protocol.py b/tests/core/test_protocol.py index 8bce162302..ffd2be8012 100644 --- a/tests/core/test_protocol.py +++ b/tests/core/test_protocol.py @@ -20,7 +20,7 @@ async def make_detector(prefix: str, name: str, tmp_path: Path): async with init_devices(mock=True): det = adsimdetector.SimDetector(prefix, dp, name=name) - det._config_sigs = [det.drv.acquire_time, det.drv.acquire] + det._config_sigs = [det.driver.acquire_time, det.driver.acquire] return det diff --git a/tests/epics/adaravis/test_aravis.py b/tests/epics/adaravis/test_aravis.py index 9333d84588..6382306d23 100644 --- a/tests/epics/adaravis/test_aravis.py +++ b/tests/epics/adaravis/test_aravis.py @@ -1,4 +1,3 @@ -import re from typing import cast import pytest @@ -26,7 +25,7 @@ async def test_deadtime_invariant_with_exposure_time( async def test_trigger_source_set_to_gpio_line(test_adaravis: adaravis.AravisDetector): - driver = cast(adaravis.AravisDriverIO, test_adaravis.drv) + driver = cast(adaravis.AravisDriverIO, test_adaravis.driver) set_mock_value(driver.trigger_source, adaravis.AravisTriggerSource.FREERUN) async def trigger_and_complete(): @@ -44,7 +43,7 @@ async def trigger_and_complete(): # Default TriggerSource assert (await driver.trigger_source.get_value()) == "Freerun" - test_adaravis._controller.set_external_trigger_gpio(1) + test_adaravis._controller.gpio_number = 1 # TriggerSource not changed by setting gpio assert (await driver.trigger_source.get_value()) == "Freerun" @@ -53,26 +52,12 @@ async def trigger_and_complete(): # TriggerSource changes assert (await driver.trigger_source.get_value()) == "Line1" - test_adaravis._controller.set_external_trigger_gpio(3) + test_adaravis._controller.gpio_number = 3 # TriggerSource not changed by setting gpio await trigger_and_complete() assert (await driver.trigger_source.get_value()) == "Line3" -def test_gpio_pin_limited(test_adaravis: adaravis.AravisDetector): - assert test_adaravis._controller.get_external_trigger_gpio() == 1 - test_adaravis._controller.set_external_trigger_gpio(2) - assert test_adaravis._controller.get_external_trigger_gpio() == 2 - with pytest.raises( - ValueError, - match=re.escape( - "AravisController only supports the following GPIO indices: " - "(1, 2, 3, 4) but was asked to use 55" - ), - ): - test_adaravis._controller.set_external_trigger_gpio(55) # type: ignore - - async def test_hints_from_hdf_writer(test_adaravis: adaravis.AravisDetector): assert test_adaravis.hints == {"fields": ["test_adaravis1"]} diff --git a/tests/epics/adcore/test_drivers.py b/tests/epics/adcore/test_drivers.py index 46fb70d8ff..333263ec22 100644 --- a/tests/epics/adcore/test_drivers.py +++ b/tests/epics/adcore/test_drivers.py @@ -6,7 +6,7 @@ from ophyd_async.core import ( init_devices, ) -from ophyd_async.epics import adcore +from ophyd_async.epics import adcore, adsimdetector from ophyd_async.testing import get_mock_put, set_mock_value TEST_DEADTIME = 0.1 @@ -20,14 +20,14 @@ def driver(RE) -> adcore.ADBaseIO: @pytest.fixture -async def controller(RE, driver: adcore.ADBaseIO) -> adcore.ADBaseController: - controller = adcore.ADBaseController(driver) +async def controller(RE, driver: adcore.ADBaseIO) -> adsimdetector.SimController: + controller = adsimdetector.SimController(driver) controller.get_deadtime = lambda exposure: TEST_DEADTIME return controller async def test_set_exposure_time_and_acquire_period_if_supplied_is_a_noop_if_no_exposure_supplied( # noqa: E501 - controller: adcore.ADBaseController, + controller: adsimdetector.SimController, driver: adcore.ADBaseIO, ): put_exposure = get_mock_put(driver.acquire_time) @@ -47,22 +47,22 @@ async def test_set_exposure_time_and_acquire_period_if_supplied_is_a_noop_if_no_ ], ) async def test_set_exposure_time_and_acquire_period_if_supplied_uses_deadtime( - controller: adcore.ADBaseController, + controller: adsimdetector.SimController, exposure: float, expected_exposure: float, expected_acquire_period: float, ): await controller.set_exposure_time_and_acquire_period_if_supplied(exposure) - actual_exposure = await controller._driver.acquire_time.get_value() - actual_acquire_period = await controller._driver.acquire_period.get_value() + actual_exposure = await controller.driver.acquire_time.get_value() + actual_acquire_period = await controller.driver.acquire_period.get_value() assert expected_exposure == actual_exposure assert expected_acquire_period == actual_acquire_period async def test_start_acquiring_driver_and_ensure_status_flags_immediate_failure( - controller: adcore.ADBaseController, + controller: adsimdetector.SimController, ): - set_mock_value(controller._driver.detector_state, adcore.DetectorState.ERROR) + set_mock_value(controller.driver.detector_state, adcore.DetectorState.ERROR) acquiring = await controller.start_acquiring_driver_and_ensure_status() with pytest.raises(ValueError): await acquiring @@ -70,19 +70,19 @@ async def test_start_acquiring_driver_and_ensure_status_flags_immediate_failure( @patch("ophyd_async.core._detector.DEFAULT_TIMEOUT", 0.2) async def test_start_acquiring_driver_and_ensure_status_fails_after_some_time( - controller: adcore.ADBaseController, + controller: adsimdetector.SimController, ): """This test ensures a failing status is captured halfway through acquisition. Real world application; it takes some time to start acquiring, and during that time the detector gets itself into a bad state. """ - set_mock_value(controller._driver.detector_state, adcore.DetectorState.IDLE) + set_mock_value(controller.driver.detector_state, adcore.DetectorState.IDLE) async def wait_then_fail(): await asyncio.sleep(0) set_mock_value( - controller._driver.detector_state, adcore.DetectorState.DISCONNECTED + controller.driver.detector_state, adcore.DetectorState.DISCONNECTED ) await wait_then_fail() diff --git a/tests/epics/adcore/test_scans.py b/tests/epics/adcore/test_scans.py index d66912aa7f..c432bd8b34 100644 --- a/tests/epics/adcore/test_scans.py +++ b/tests/epics/adcore/test_scans.py @@ -18,7 +18,7 @@ TriggerInfo, init_devices, ) -from ophyd_async.epics import adcore +from ophyd_async.epics import adcore, adsimdetector from ophyd_async.testing import set_mock_value @@ -53,11 +53,11 @@ def get_deadtime(self, exposure: float | None) -> float: @pytest.fixture -def controller(RE) -> adcore.ADBaseController: +def controller(RE) -> adsimdetector.SimController: with init_devices(mock=True): drv = adcore.ADBaseIO("DRV") - return adcore.ADBaseController(drv) + return adsimdetector.SimController(drv) @pytest.fixture @@ -78,9 +78,9 @@ def writer(RE, static_path_provider, tmp_path: Path) -> adcore.ADHDFWriter: async def test_hdf_writer_fails_on_timeout_with_stepscan( RE: RunEngine, writer: adcore.ADHDFWriter, - controller: adcore.ADBaseController, + controller: adsimdetector.SimController, ): - set_mock_value(writer._fileio.file_path_exists, True) + set_mock_value(writer.fileio.file_path_exists, True) detector: StandardDetector[Any, Any] = StandardDetector( controller, writer, name="detector" ) @@ -96,7 +96,7 @@ def test_hdf_writer_fails_on_timeout_with_flyscan( RE: RunEngine, writer: adcore.ADHDFWriter ): controller = DummyController() - set_mock_value(writer._fileio.file_path_exists, True) + set_mock_value(writer.fileio.file_path_exists, True) detector: StandardDetector[Any, Any] = StandardDetector(controller, writer) trigger_logic = DummyTriggerLogic() diff --git a/tests/epics/adcore/test_writers.py b/tests/epics/adcore/test_writers.py index c5922ee23d..39c56e29f9 100644 --- a/tests/epics/adcore/test_writers.py +++ b/tests/epics/adcore/test_writers.py @@ -107,7 +107,7 @@ async def test_stats_describe_when_plugin_configured( hdf_writer_with_stats: adcore.ADHDFWriter, ): assert hdf_writer_with_stats._file is None - set_mock_value(hdf_writer_with_stats._fileio.file_path_exists, True) + set_mock_value(hdf_writer_with_stats.fileio.file_path_exists, True) assert hdf_writer_with_stats._plugins is not None set_mock_value( hdf_writer_with_stats._plugins["stats"].nd_attributes_file, @@ -160,7 +160,7 @@ async def test_stats_describe_raises_error_with_dbr_native( hdf_writer_with_stats: adcore.ADHDFWriter, ): assert hdf_writer_with_stats._file is None - set_mock_value(hdf_writer_with_stats._fileio.file_path_exists, True) + set_mock_value(hdf_writer_with_stats.fileio.file_path_exists, True) assert hdf_writer_with_stats._plugins is not None set_mock_value( hdf_writer_with_stats._plugins["stats"].nd_attributes_file, diff --git a/tests/epics/adkinetix/test_kinetix.py b/tests/epics/adkinetix/test_kinetix.py index 27bedf3394..19e0116b4a 100644 --- a/tests/epics/adkinetix/test_kinetix.py +++ b/tests/epics/adkinetix/test_kinetix.py @@ -24,7 +24,7 @@ async def test_get_deadtime( async def test_trigger_modes(test_adkinetix: adkinetix.KinetixDetector): - driver = cast(adkinetix.KinetixDriverIO, test_adkinetix.drv) + driver = cast(adkinetix.KinetixDriverIO, test_adkinetix.driver) set_mock_value(driver.trigger_mode, adkinetix.KinetixTriggerMode.INTERNAL) async def setup_trigger_mode(trig_mode: DetectorTrigger): diff --git a/tests/epics/adpilatus/test_pilatus.py b/tests/epics/adpilatus/test_pilatus.py index e715b3e5fe..e5c8613e03 100644 --- a/tests/epics/adpilatus/test_pilatus.py +++ b/tests/epics/adpilatus/test_pilatus.py @@ -49,7 +49,7 @@ async def test_trigger_mode_set( expected_trigger_mode: adpilatus.PilatusTriggerMode, ): async def trigger_and_complete(): - set_mock_value(test_adpilatus.drv.armed, True) + set_mock_value(test_adpilatus.driver.armed, True) await test_adpilatus._controller.prepare( TriggerInfo(number_of_triggers=1, trigger=detector_trigger) ) @@ -86,7 +86,7 @@ async def _trigger( expected_trigger_mode: adpilatus.PilatusTriggerMode, trigger_and_complete: Callable[[], Awaitable], ): - pilatus_driver = test_adpilatus.drv + pilatus_driver = test_adpilatus.driver # Default TriggerMode assert ( await pilatus_driver.trigger_mode.get_value() @@ -125,7 +125,7 @@ async def dummy_open(multiplier: int = 0): return {} test_adpilatus._writer.open = dummy_open - set_mock_value(test_adpilatus.drv.armed, True) + set_mock_value(test_adpilatus.driver.armed, True) await test_adpilatus.prepare( TriggerInfo( number_of_triggers=1, @@ -134,13 +134,13 @@ async def dummy_open(multiplier: int = 0): livetime=1.0, ) ) - assert (await test_adpilatus.drv.acquire_time.get_value()) == 1.0 - assert (await test_adpilatus.drv.acquire_period.get_value()) == 1.0 + 950e-6 + assert (await test_adpilatus.driver.acquire_time.get_value()) == 1.0 + assert (await test_adpilatus.driver.acquire_period.get_value()) == 1.0 + 950e-6 async def test_pilatus_controller(test_adpilatus: adpilatus.PilatusDetector): pilatus = test_adpilatus._controller - pilatus_driver = cast(adpilatus.PilatusDriverIO, test_adpilatus.drv) + pilatus_driver = cast(adpilatus.PilatusDriverIO, test_adpilatus.driver) set_mock_value(pilatus_driver.armed, True) await pilatus.prepare( TriggerInfo(number_of_triggers=1, trigger=DetectorTrigger.CONSTANT_GATE) diff --git a/tests/epics/adsimdetector/test_sim.py b/tests/epics/adsimdetector/test_sim.py index 7cbcdd6d95..11c06b9dd5 100644 --- a/tests/epics/adsimdetector/test_sim.py +++ b/tests/epics/adsimdetector/test_sim.py @@ -54,7 +54,7 @@ def count_sim(dets: Sequence[adsimdetector.SimDetector], times: int = 1): for _ in range(times): read_values = {} for det in dets: - read_values[det] = yield from bps.rd(det._writer._fileio.num_captured) + read_values[det] = yield from bps.rd(det._writer.fileio.num_captured) for det in dets: yield from bps.trigger(det, wait=False, group="wait_for_trigger") @@ -62,7 +62,7 @@ def count_sim(dets: Sequence[adsimdetector.SimDetector], times: int = 1): yield from bps.sleep(1.0) [ set_mock_value( - det._writer._fileio.num_captured, + det._writer.fileio.num_captured, read_values[det] + 1, ) for det in dets @@ -154,7 +154,7 @@ async def test_two_detectors_step( RE.subscribe(lambda name, _: names.append(name)) RE.subscribe(lambda _, doc: docs.append(doc)) [ - set_mock_value(det._writer._fileio.file_path_exists, True) + set_mock_value(det.fileio.file_path_exists, True) for det in two_test_adsimdetectors ] @@ -170,26 +170,26 @@ def plan(): nonlocal file_name_a, file_name_b yield from count_sim(two_test_adsimdetectors, times=1) - drv = controller_a._driver + drv = controller_a.driver assert False is (yield from bps.rd(drv.acquire)) assert adcore.ImageMode.MULTIPLE == (yield from bps.rd(drv.image_mode)) - hdfb = cast(adcore.NDFileHDFIO, writer_b._fileio) + hdfb = cast(adcore.NDFileHDFIO, writer_b.fileio) assert True is (yield from bps.rd(hdfb.lazy_open)) assert True is (yield from bps.rd(hdfb.swmr_mode)) assert 0 == (yield from bps.rd(hdfb.num_capture)) assert adcore.FileWriteMode.STREAM == (yield from bps.rd(hdfb.file_write_mode)) - assert (yield from bps.rd(writer_a._fileio.file_path)) == str( + assert (yield from bps.rd(writer_a.fileio.file_path)) == str( info_a.directory_path ) - file_name_a = yield from bps.rd(writer_a._fileio.file_name) + file_name_a = yield from bps.rd(writer_a.fileio.file_name) assert file_name_a == info_a.filename - assert (yield from bps.rd(writer_b._fileio.file_path)) == str( + assert (yield from bps.rd(writer_b.fileio.file_path)) == str( info_b.directory_path ) - file_name_b = yield from bps.rd(writer_b._fileio.file_name) + file_name_b = yield from bps.rd(writer_b.fileio.file_name) assert file_name_b == info_b.filename RE(plan()) @@ -206,10 +206,10 @@ def plan(): _, descriptor, sra, sda, srb, sdb, event, _ = docs assert descriptor["configuration"]["test_adsim1"]["data"][ - "test_adsim1-drv-acquire_time" + "test_adsim1-driver-acquire_time" ] == pytest.approx(0.8) assert descriptor["configuration"]["test_adsim2"]["data"][ - "test_adsim2-drv-acquire_time" + "test_adsim2-driver-acquire_time" ] == pytest.approx(1.8) assert descriptor["data_keys"]["test_adsim1"]["shape"] == [10, 10] assert descriptor["data_keys"]["test_adsim2"]["shape"] == [11, 11] @@ -247,15 +247,13 @@ async def test_detector_writes_to_file( RE.subscribe(lambda name, _: names.append(name)) RE.subscribe(lambda _, doc: docs.append(doc)) set_mock_value( - test_adsimdetector._writer._fileio.file_path_exists, + test_adsimdetector.fileio.file_path_exists, True, ) RE(count_sim([test_adsimdetector], times=3)) - assert await test_adsimdetector._writer._fileio.file_path.get_value() == str( - tmp_path - ) + assert await test_adsimdetector.fileio.file_path.get_value() == str(tmp_path) descriptor_index = names.index("descriptor") @@ -285,13 +283,13 @@ async def test_read_and_describe_detector( describe = await test_adsimdetector.describe_configuration() read = await test_adsimdetector.read_configuration() assert describe == { - "test_adsim1-drv-acquire_time": { + "test_adsim1-driver-acquire_time": { "source": "mock+ca://SIM1:cam1:AcquireTime_RBV", "dtype": "number", "dtype_numpy": " Date: Mon, 23 Dec 2024 11:17:29 -0500 Subject: [PATCH 38/60] Resolve unawaited coro error on 3.12 --- src/ophyd_async/epics/adaravis/_aravis_controller.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ophyd_async/epics/adaravis/_aravis_controller.py b/src/ophyd_async/epics/adaravis/_aravis_controller.py index f847d52a66..b2daa7922e 100644 --- a/src/ophyd_async/epics/adaravis/_aravis_controller.py +++ b/src/ophyd_async/epics/adaravis/_aravis_controller.py @@ -38,7 +38,7 @@ async def prepare(self, trigger_info: TriggerInfo): else: image_mode = adcore.ImageMode.MULTIPLE if (exposure := trigger_info.livetime) is not None: - await self.driver.acquire_time.set(exposure) + asyncio.gather(self.driver.acquire_time.set(exposure)) trigger_mode, trigger_source = self._get_trigger_info(trigger_info.trigger) # trigger mode must be set first and on it's own! From 7a860842de5eff446de36ac1e9616159b9c809b9 Mon Sep 17 00:00:00 2001 From: thomashopkins32 Date: Thu, 26 Dec 2024 10:12:12 -0500 Subject: [PATCH 39/60] Finish changing multiplier -> batch_size; Fix some untyped imports --- src/ophyd_async/core/_detector.py | 4 ++-- src/ophyd_async/core/_hdf_dataset.py | 6 +++--- src/ophyd_async/epics/adcore/_core_writer.py | 7 +++---- src/ophyd_async/epics/adcore/_hdf_writer.py | 12 ++++++------ src/ophyd_async/epics/eiger/_odin_io.py | 5 +++-- src/ophyd_async/fastcs/panda/_writer.py | 18 +++++++++--------- .../_pattern_detector_writer.py | 4 ++-- .../_pattern_detector/_pattern_generator.py | 14 ++++++-------- tests/core/test_flyer.py | 11 ++++++----- tests/epics/adaravis/test_aravis.py | 2 +- tests/epics/adkinetix/test_kinetix.py | 2 +- tests/epics/adpilatus/test_pilatus.py | 2 +- tests/epics/advimba/test_vimba.py | 2 +- tests/fastcs/panda/test_hdf_panda.py | 2 +- tests/fastcs/panda/test_writer.py | 4 ++-- tests/plan_stubs/test_fly.py | 11 ++++++----- 16 files changed, 53 insertions(+), 53 deletions(-) diff --git a/src/ophyd_async/core/_detector.py b/src/ophyd_async/core/_detector.py index 96b2c3bfcb..eece95b05c 100644 --- a/src/ophyd_async/core/_detector.py +++ b/src/ophyd_async/core/_detector.py @@ -21,7 +21,7 @@ Triggerable, WritesStreamAssets, ) -from event_model import DataKey +from event_model import DataKey # type: ignore from pydantic import BaseModel, Field, NonNegativeInt, computed_field from ._device import Device, DeviceConnector @@ -65,7 +65,7 @@ class TriggerInfo(BaseModel): livetime: float | None = Field(default=None, ge=0) #: What is the maximum timeout on waiting for a frame frame_timeout: float | None = Field(default=None, gt=0) - #: The number of triggers are grouped into a single StreamDatum index. + #: The number of triggers that are grouped into a single StreamDatum index. #: A batch_size > 1 can be useful to have frames from a faster detector able to be zipped with a single frame from a slower detector. #: E.g. if number_of_triggers=10 and batch_size=5 then the detector will take 10 frames, #: but publish 2 StreamDatum indices, and describe() will show a shape of (5, h, w) for each. diff --git a/src/ophyd_async/core/_hdf_dataset.py b/src/ophyd_async/core/_hdf_dataset.py index 03289b2579..7b465cab1b 100644 --- a/src/ophyd_async/core/_hdf_dataset.py +++ b/src/ophyd_async/core/_hdf_dataset.py @@ -3,7 +3,7 @@ from pathlib import Path from urllib.parse import urlunparse -from event_model import ( +from event_model import ( # type: ignore ComposeStreamResource, ComposeStreamResourceBundle, StreamDatum, @@ -18,7 +18,7 @@ class HDFDataset: dataset: str shape: Sequence[int] = field(default_factory=tuple) dtype_numpy: str = "" - frequency_ratio: int = 1 + batch_size: int = 1 swmr: bool = False # Represents explicit chunk size written to disk. chunk_shape: tuple[int, ...] = () @@ -67,7 +67,7 @@ def __init__( parameters={ "dataset": ds.dataset, "swmr": ds.swmr, - "group_factor": ds.group_factor, + "batch_size": ds.batch_size, "chunk_shape": ds.chunk_shape, }, uid=None, diff --git a/src/ophyd_async/epics/adcore/_core_writer.py b/src/ophyd_async/epics/adcore/_core_writer.py index 5be2fa842f..48bf9859d0 100644 --- a/src/ophyd_async/epics/adcore/_core_writer.py +++ b/src/ophyd_async/epics/adcore/_core_writer.py @@ -5,7 +5,7 @@ from urllib.parse import urlunparse from bluesky.protocols import Hints, StreamAsset -from event_model import ( +from event_model import ( # type: ignore ComposeStreamResource, DataKey, StreamRange, @@ -148,11 +148,11 @@ async def observe_indices_written( ) -> AsyncGenerator[int, None]: """Wait until a specific index is ready to be collected""" async for num_captured in observe_value(self.fileio.num_captured, timeout): - yield num_captured // self._multiplier + yield num_captured // self._batch_size async def get_indices_written(self) -> int: num_captured = await self.fileio.num_captured.get_value() - return num_captured // self._multiplier + return num_captured // self._batch_size async def collect_stream_docs( self, indices_written: int @@ -183,7 +183,6 @@ async def collect_stream_docs( uri=uri, data_key=self._name_provider(), parameters={ - # TODO: Is the following assumption accurate or should this be `self._batch_size`? # Assume that we always write 1 frame per file/chunk "chunk_shape": (1, *frame_shape), # Include file template for reconstruction in consolidator diff --git a/src/ophyd_async/epics/adcore/_hdf_writer.py b/src/ophyd_async/epics/adcore/_hdf_writer.py index 7e93654e3b..76601fb4bd 100644 --- a/src/ophyd_async/epics/adcore/_hdf_writer.py +++ b/src/ophyd_async/epics/adcore/_hdf_writer.py @@ -4,7 +4,7 @@ from xml.etree import ElementTree as ET from bluesky.protocols import Hints, StreamAsset -from event_model import DataKey +from event_model import DataKey # type: ignore from ophyd_async.core import ( DEFAULT_TIMEOUT, @@ -48,7 +48,7 @@ def __init__( self._file: HDFFile | None = None self._include_file_number = False - async def open(self, frequency_ratio: int = 1) -> dict[str, DataKey]: + async def open(self, batch_size: int = 1) -> dict[str, DataKey]: self._file = None # Setting HDF writer specific signals @@ -74,7 +74,7 @@ async def open(self, frequency_ratio: int = 1) -> dict[str, DataKey]: name = self._name_provider() detector_shape = await self._dataset_describer.shape() np_dtype = await self._dataset_describer.np_datatype() - self._frequency_ratio = frequency_ratio + self._batch_size = batch_size # Determine number of frames that will be saved per HDF chunk frames_per_chunk = await self.fileio.num_frames_chunks.get_value() @@ -86,7 +86,7 @@ async def open(self, frequency_ratio: int = 1) -> dict[str, DataKey]: dataset="/entry/data/data", shape=detector_shape, dtype_numpy=np_dtype, - frequency_ratio=frequency_ratio, + batch_size=batch_size, chunk_shape=(frames_per_chunk, *detector_shape), ) ] @@ -113,7 +113,7 @@ async def open(self, frequency_ratio: int = 1) -> dict[str, DataKey]: f"/entry/instrument/NDAttributes/{datakey}", (), np_datatype, - multiplier, + batch_size, # NDAttributes appear to always be configured with # this chunk size chunk_shape=(16384,), @@ -123,7 +123,7 @@ async def open(self, frequency_ratio: int = 1) -> dict[str, DataKey]: describe = { ds.data_key: DataKey( source=self.fileio.full_file_name.source, - shape=list(outer_shape + tuple(ds.shape)), + shape=list((batch_size, *ds.shape)), dtype="array" if ds.shape else "number", dtype_numpy=ds.dtype_numpy, external="STREAM:", diff --git a/src/ophyd_async/epics/eiger/_odin_io.py b/src/ophyd_async/epics/eiger/_odin_io.py index 19e0a0fc4e..c051fe3549 100644 --- a/src/ophyd_async/epics/eiger/_odin_io.py +++ b/src/ophyd_async/epics/eiger/_odin_io.py @@ -77,8 +77,9 @@ def __init__( self._name_provider = name_provider super().__init__() - async def open(self, multiplier: int = 1) -> dict[str, DataKey]: + async def open(self, batch_size: int = 1) -> dict[str, DataKey]: info = self._path_provider(device_name=self._name_provider()) + self._batch_size = batch_size await asyncio.gather( self._drv.file_path.set(str(info.directory_path)), @@ -101,7 +102,7 @@ async def _describe(self) -> dict[str, DataKey]: return { "data": DataKey( source=self._drv.file_name.source, - shape=list(data_shape), + shape=list((self._batch_size, *data_shape)), dtype="array", # TODO: Use correct type based on eiger https://github.com/bluesky/ophyd-async/issues/529 dtype_numpy=" dict[str, DataKey]: + async def open(self, batch_size: int = 1) -> dict[str, DataKey]: """Retrieve and get descriptor of all PandA signals marked for capture""" # Ensure flushes are immediate @@ -68,9 +68,9 @@ async def open(self, multiplier: int = 1) -> dict[str, DataKey]: # Wait for it to start, stashing the status that tells us when it finishes await self.panda_data_block.capture.set(True) - if multiplier > 1: + if batch_size > 1: raise ValueError( - "All PandA datasets should be scalar, multiplier should be 1" + "All PandA datasets should be scalar, batch_size should be 1" ) return await self._describe() @@ -84,7 +84,7 @@ async def _describe(self) -> dict[str, DataKey]: describe = { ds.data_key: DataKey( source=self.panda_data_block.hdf_directory.source, - shape=list(ds.shape), + shape=list((self._batch_size, *ds.shape)), dtype="array" if ds.shape != [1] else "number", # PandA data should always be written as Float64 dtype_numpy=" None: # TODO: Update chunk size to read signal once available in IOC # Currently PandA IOC sets chunk size to 1024 points per chunk HDFDataset( - dataset_name, "/" + dataset_name, [1], multiplier=1, chunk_shape=(1024,) + dataset_name, "/" + dataset_name, [1], batch_size=1, chunk_shape=(1024,) ) for dataset_name in capture_table.name ] @@ -141,7 +141,7 @@ async def observe_indices_written( async for num_captured in observe_value( self.panda_data_block.num_captured, timeout ): - yield num_captured // self._multiplier + yield num_captured // self._batch_size async def collect_stream_docs( self, indices_written: int diff --git a/src/ophyd_async/sim/_pattern_detector/_pattern_detector_writer.py b/src/ophyd_async/sim/_pattern_detector/_pattern_detector_writer.py index 16dda6f69f..cfc132f878 100644 --- a/src/ophyd_async/sim/_pattern_detector/_pattern_detector_writer.py +++ b/src/ophyd_async/sim/_pattern_detector/_pattern_detector_writer.py @@ -20,9 +20,9 @@ def __init__( self.path_provider = path_provider self.name_provider = name_provider - async def open(self, multiplier: int = 1) -> dict[str, DataKey]: + async def open(self, batch_size: int = 1) -> dict[str, DataKey]: return await self.pattern_generator.open_file( - self.path_provider, self.name_provider(), multiplier + self.path_provider, self.name_provider(), batch_size ) async def close(self) -> None: diff --git a/src/ophyd_async/sim/_pattern_detector/_pattern_generator.py b/src/ophyd_async/sim/_pattern_detector/_pattern_generator.py index 01c54c8245..5c9e3f4cb8 100644 --- a/src/ophyd_async/sim/_pattern_detector/_pattern_generator.py +++ b/src/ophyd_async/sim/_pattern_detector/_pattern_generator.py @@ -105,7 +105,7 @@ def set_y(self, value: float) -> None: self.y = value async def open_file( - self, path_provider: PathProvider, name: str, multiplier: int = 1 + self, path_provider: PathProvider, name: str, batch_size: int = 1 ) -> dict[str, DataKey]: await self.counter_signal.connect() @@ -131,9 +131,7 @@ async def open_file( # once datasets written, can switch the model to single writer multiple reader self._handle_for_h5_file.swmr_mode = True - self.multiplier = multiplier - - outer_shape = (multiplier,) if multiplier > 1 else () + self._batch_size = batch_size # cache state to self # Add the main data @@ -142,20 +140,20 @@ async def open_file( data_key=name, dataset=DATA_PATH, shape=(self.height, self.width), - multiplier=multiplier, + batch_size=batch_size, ), HDFDataset( f"{name}-sum", dataset=SUM_PATH, shape=(), - multiplier=multiplier, + batch_size=batch_size, ), ] describe = { ds.data_key: DataKey( source="sim://pattern-generator-hdf-file", - shape=list(outer_shape) + list(ds.shape), + shape=list((self._batch_size, *ds.shape)), dtype="array" if ds.shape else "number", external="STREAM:", ) @@ -204,4 +202,4 @@ async def observe_indices_written( self, timeout=DEFAULT_TIMEOUT ) -> AsyncGenerator[int, None]: async for num_captured in observe_value(self.counter_signal, timeout=timeout): - yield num_captured // self.multiplier + yield num_captured // self._batch_size diff --git a/tests/core/test_flyer.py b/tests/core/test_flyer.py index ef16a53c51..cf23695059 100644 --- a/tests/core/test_flyer.py +++ b/tests/core/test_flyer.py @@ -62,11 +62,12 @@ def __init__(self, name: str, shape: Sequence[int]): self._last_emitted = 0 self.index = 0 - async def open(self, multiplier: int = 1) -> dict[str, DataKey]: + async def open(self, batch_size: int = 1) -> dict[str, DataKey]: + self._batch_size = batch_size return { self._name: DataKey( source="soft://some-source", - shape=self._shape, + shape=list((batch_size, *self._shape)), dtype="number", dtype_numpy=" AsyncGenerator[int, None]: num_captured: int async for num_captured in observe_value(self.dummy_signal, timeout): - yield num_captured + yield num_captured // self._batch_size async def get_indices_written(self) -> int: - return self.index + return self.index // self._batch_size async def collect_stream_docs( self, indices_written: int @@ -95,7 +96,7 @@ async def collect_stream_docs( parameters={ "path": "", "dataset": "", - "multiplier": False, + "batch_size": self._batch_size, }, uid=None, validate=True, diff --git a/tests/epics/adaravis/test_aravis.py b/tests/epics/adaravis/test_aravis.py index 6382306d23..9ce5a5f119 100644 --- a/tests/epics/adaravis/test_aravis.py +++ b/tests/epics/adaravis/test_aravis.py @@ -105,7 +105,7 @@ async def test_can_collect( assert stream_resource["parameters"] == { "dataset": "/entry/data/data", "swmr": False, - "multiplier": 1, + "batch_size": 1, "chunk_shape": (1, 10, 10), } assert docs[1][0] == "stream_datum" diff --git a/tests/epics/adkinetix/test_kinetix.py b/tests/epics/adkinetix/test_kinetix.py index 19e0116b4a..c0deb40117 100644 --- a/tests/epics/adkinetix/test_kinetix.py +++ b/tests/epics/adkinetix/test_kinetix.py @@ -100,7 +100,7 @@ async def test_can_collect( assert stream_resource["parameters"] == { "dataset": "/entry/data/data", "swmr": False, - "multiplier": 1, + "batch_size": 1, "chunk_shape": (1, 10, 10), } assert docs[1][0] == "stream_datum" diff --git a/tests/epics/adpilatus/test_pilatus.py b/tests/epics/adpilatus/test_pilatus.py index e5c8613e03..fce84c7d92 100644 --- a/tests/epics/adpilatus/test_pilatus.py +++ b/tests/epics/adpilatus/test_pilatus.py @@ -121,7 +121,7 @@ async def test_unsupported_trigger_excepts(test_adpilatus: adpilatus.PilatusDete async def test_exposure_time_and_acquire_period_set( test_adpilatus: adpilatus.PilatusDetector, ): - async def dummy_open(multiplier: int = 0): + async def dummy_open(batch_size: int = 0): return {} test_adpilatus._writer.open = dummy_open diff --git a/tests/epics/advimba/test_vimba.py b/tests/epics/advimba/test_vimba.py index 541cbc60f8..dc1e8454b7 100644 --- a/tests/epics/advimba/test_vimba.py +++ b/tests/epics/advimba/test_vimba.py @@ -118,7 +118,7 @@ async def test_can_collect( assert stream_resource["parameters"] == { "dataset": "/entry/data/data", "swmr": False, - "multiplier": 1, + "batch_size": 1, "chunk_shape": (1, 10, 10), } assert docs[1][0] == "stream_datum" diff --git a/tests/fastcs/panda/test_hdf_panda.py b/tests/fastcs/panda/test_hdf_panda.py index d56af3663e..5624639e03 100644 --- a/tests/fastcs/panda/test_hdf_panda.py +++ b/tests/fastcs/panda/test_hdf_panda.py @@ -146,7 +146,7 @@ def flying_plan(): "parameters": { "dataset": f"/{dataset_name}", "swmr": False, - "multiplier": 1, + "batch_size": 1, "chunk_shape": (1024,), }, } diff --git a/tests/fastcs/panda/test_writer.py b/tests/fastcs/panda/test_writer.py index f184ceaf07..fa60b15ab9 100644 --- a/tests/fastcs/panda/test_writer.py +++ b/tests/fastcs/panda/test_writer.py @@ -149,7 +149,7 @@ async def test_open_sets_file_path_and_name(mock_writer: PandaHDFWriter, tmp_pat assert name == "data.h5" -async def test_open_errors_when_multiplier_not_one(mock_writer: PandaHDFWriter): +async def test_open_errors_when_batch_size_not_one(mock_writer: PandaHDFWriter): with pytest.raises(ValueError): await mock_writer.open(2) @@ -191,7 +191,7 @@ def assert_resource_document(name, resource_doc): "parameters": { "dataset": f"/{name}", "swmr": False, - "multiplier": 1, + "batch_size": 1, "chunk_shape": (1024,), }, } diff --git a/tests/plan_stubs/test_fly.py b/tests/plan_stubs/test_fly.py index 48a82e34bb..e48918b83d 100644 --- a/tests/plan_stubs/test_fly.py +++ b/tests/plan_stubs/test_fly.py @@ -47,11 +47,12 @@ def __init__(self, name: str, shape: Sequence[int]): self.index = 0 self.observe_indices_written_timeout_log = [] - async def open(self, multiplier: int = 1) -> dict[str, DataKey]: + async def open(self, batch_size: int = 1) -> dict[str, DataKey]: + self._batch_size = batch_size return { self._name: DataKey( source="soft://some-source", - shape=self._shape, + shape=list((batch_size, *self._shape)), dtype="number", external="STREAM:", ) @@ -63,10 +64,10 @@ async def observe_indices_written( self.observe_indices_written_timeout_log.append(timeout) num_captured: int async for num_captured in observe_value(self.dummy_signal, timeout): - yield num_captured + yield num_captured // self._batch_size async def get_indices_written(self) -> int: - return self.index + return self.index // self._batch_size async def collect_stream_docs( self, indices_written: int @@ -80,7 +81,7 @@ async def collect_stream_docs( parameters={ "path": "", "swmr": False, - "multiplier": 1, + "batch_size": 1, }, uid=None, validate=True, From ed02c6b96a8e90aecfd2fcbc74c527bfdcdc8f37 Mon Sep 17 00:00:00 2001 From: thomashopkins32 Date: Thu, 26 Dec 2024 10:48:59 -0500 Subject: [PATCH 40/60] Add more type ignores --- src/ophyd_async/core/_detector.py | 2 +- src/ophyd_async/core/_hdf_dataset.py | 2 +- src/ophyd_async/epics/adcore/_core_writer.py | 4 ++-- src/ophyd_async/epics/adcore/_hdf_writer.py | 2 +- src/ophyd_async/fastcs/panda/_writer.py | 4 ++-- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/ophyd_async/core/_detector.py b/src/ophyd_async/core/_detector.py index eece95b05c..50117b8d8e 100644 --- a/src/ophyd_async/core/_detector.py +++ b/src/ophyd_async/core/_detector.py @@ -21,7 +21,7 @@ Triggerable, WritesStreamAssets, ) -from event_model import DataKey # type: ignore +from event_model import DataKey # type: ignore from pydantic import BaseModel, Field, NonNegativeInt, computed_field from ._device import Device, DeviceConnector diff --git a/src/ophyd_async/core/_hdf_dataset.py b/src/ophyd_async/core/_hdf_dataset.py index 7b465cab1b..ab08a09177 100644 --- a/src/ophyd_async/core/_hdf_dataset.py +++ b/src/ophyd_async/core/_hdf_dataset.py @@ -3,7 +3,7 @@ from pathlib import Path from urllib.parse import urlunparse -from event_model import ( # type: ignore +from event_model import ( # type: ignore ComposeStreamResource, ComposeStreamResourceBundle, StreamDatum, diff --git a/src/ophyd_async/epics/adcore/_core_writer.py b/src/ophyd_async/epics/adcore/_core_writer.py index 48bf9859d0..19fbd2896f 100644 --- a/src/ophyd_async/epics/adcore/_core_writer.py +++ b/src/ophyd_async/epics/adcore/_core_writer.py @@ -5,7 +5,7 @@ from urllib.parse import urlunparse from bluesky.protocols import Hints, StreamAsset -from event_model import ( # type: ignore +from event_model import ( # type: ignore ComposeStreamResource, DataKey, StreamRange, @@ -135,7 +135,7 @@ async def open(self, batch_size: int = 1) -> dict[str, DataKey]: describe = { self._name_provider(): DataKey( source=self._name_provider(), - shape=list((batch_size,) + frame_shape), + shape=list((batch_size, *frame_shape)), dtype="array", dtype_numpy=dtype_numpy, external="STREAM:", diff --git a/src/ophyd_async/epics/adcore/_hdf_writer.py b/src/ophyd_async/epics/adcore/_hdf_writer.py index 76601fb4bd..91ff97ec55 100644 --- a/src/ophyd_async/epics/adcore/_hdf_writer.py +++ b/src/ophyd_async/epics/adcore/_hdf_writer.py @@ -4,7 +4,7 @@ from xml.etree import ElementTree as ET from bluesky.protocols import Hints, StreamAsset -from event_model import DataKey # type: ignore +from event_model import DataKey # type: ignore from ophyd_async.core import ( DEFAULT_TIMEOUT, diff --git a/src/ophyd_async/fastcs/panda/_writer.py b/src/ophyd_async/fastcs/panda/_writer.py index a2cc178384..4037582089 100644 --- a/src/ophyd_async/fastcs/panda/_writer.py +++ b/src/ophyd_async/fastcs/panda/_writer.py @@ -3,8 +3,8 @@ from pathlib import Path from bluesky.protocols import StreamAsset -from event_model import DataKey # type: ignore -from p4p.client.thread import Context # type: ignore +from event_model import DataKey # type: ignore +from p4p.client.thread import Context # type: ignore from ophyd_async.core import ( DEFAULT_TIMEOUT, From cc944f93e0a6e669a07946b13bcfe696f505f5b5 Mon Sep 17 00:00:00 2001 From: thomashopkins32 Date: Thu, 26 Dec 2024 11:33:40 -0500 Subject: [PATCH 41/60] Add extra first dimension to tests' outputs --- tests/core/test_flyer.py | 1 + tests/epics/adaravis/test_aravis.py | 4 ++-- tests/epics/adcore/test_writers.py | 6 +++--- tests/epics/adkinetix/test_kinetix.py | 4 ++-- tests/epics/adsimdetector/test_sim.py | 5 +++-- tests/epics/advimba/test_vimba.py | 4 ++-- tests/epics/eiger/test_odin_io.py | 2 +- tests/fastcs/panda/test_writer.py | 2 +- tests/plan_stubs/test_fly.py | 1 + tests/sim/test_sim_writer.py | 4 ++-- 10 files changed, 18 insertions(+), 15 deletions(-) diff --git a/tests/core/test_flyer.py b/tests/core/test_flyer.py index cf23695059..f9931b5e50 100644 --- a/tests/core/test_flyer.py +++ b/tests/core/test_flyer.py @@ -60,6 +60,7 @@ def __init__(self, name: str, shape: Sequence[int]): self._name = name self._file: ComposeStreamResourceBundle | None = None self._last_emitted = 0 + self._batch_size = 1 self.index = 0 async def open(self, batch_size: int = 1) -> dict[str, DataKey]: diff --git a/tests/epics/adaravis/test_aravis.py b/tests/epics/adaravis/test_aravis.py index 9ce5a5f119..aa8f097451 100644 --- a/tests/epics/adaravis/test_aravis.py +++ b/tests/epics/adaravis/test_aravis.py @@ -76,7 +76,7 @@ async def test_decribe_describes_writer_dataset( assert await test_adaravis.describe() == { "test_adaravis1": { "source": "mock+ca://ARAVIS1:HDF1:FullFileName_RBV", - "shape": [10, 10], + "shape": [1, 10, 10], "dtype": "array", "dtype_numpy": "|i1", "external": "STREAM:", @@ -124,7 +124,7 @@ async def test_can_decribe_collect( assert (await test_adaravis.describe_collect()) == { "test_adaravis1": { "source": "mock+ca://ARAVIS1:HDF1:FullFileName_RBV", - "shape": [10, 10], + "shape": [1, 10, 10], "dtype": "array", "dtype_numpy": "|i1", "external": "STREAM:", diff --git a/tests/epics/adcore/test_writers.py b/tests/epics/adcore/test_writers.py index 39c56e29f9..6e5fa46441 100644 --- a/tests/epics/adcore/test_writers.py +++ b/tests/epics/adcore/test_writers.py @@ -133,14 +133,14 @@ async def test_stats_describe_when_plugin_configured( assert descriptor == { "test": { "source": "mock+ca://HDF:FullFileName_RBV", - "shape": [10, 10], + "shape": [1, 10, 10], "dtype": "array", "dtype_numpy": " Date: Thu, 26 Dec 2024 16:42:18 -0500 Subject: [PATCH 42/60] Added some questions and TODOs that need to be resolved; Otherwise, finished moving the multiplier into the DataKey.shape (needs testing) --- src/ophyd_async/core/_detector.py | 3 +++ src/ophyd_async/epics/adcore/_core_writer.py | 14 +++++++------- src/ophyd_async/epics/eiger/_odin_io.py | 6 +++--- src/ophyd_async/fastcs/panda/_writer.py | 6 +++--- .../_pattern_detector/_pattern_detector_writer.py | 8 +++++--- tests/core/test_flyer.py | 6 +++--- tests/plan_stubs/test_fly.py | 6 +++--- 7 files changed, 27 insertions(+), 22 deletions(-) diff --git a/src/ophyd_async/core/_detector.py b/src/ophyd_async/core/_detector.py index 50117b8d8e..5ca10bf9e5 100644 --- a/src/ophyd_async/core/_detector.py +++ b/src/ophyd_async/core/_detector.py @@ -329,6 +329,9 @@ async def prepare(self, value: TriggerInfo) -> None: if isinstance(self._trigger_info.number_of_triggers, list) else [self._trigger_info.number_of_triggers] ) + # TODO: How can we get the indices written prior to opening the writer? + # This is a problem since the `get_indices_written` needs to know the batch_size + # to return the correct number of indices written. self._initial_frame = await self._writer.get_indices_written() self._describe, _ = await asyncio.gather( self._writer.open(value.batch_size), self._controller.prepare(value) diff --git a/src/ophyd_async/epics/adcore/_core_writer.py b/src/ophyd_async/epics/adcore/_core_writer.py index 19fbd2896f..5cb08fa0a1 100644 --- a/src/ophyd_async/epics/adcore/_core_writer.py +++ b/src/ophyd_async/epics/adcore/_core_writer.py @@ -5,11 +5,7 @@ from urllib.parse import urlunparse from bluesky.protocols import Hints, StreamAsset -from event_model import ( # type: ignore - ComposeStreamResource, - DataKey, - StreamRange, -) +from event_model import DataKey, ComposeStreamResource, StreamRange # type: ignore from ophyd_async.core._detector import DetectorWriter from ophyd_async.core._providers import DatasetDescriber, NameProvider, PathProvider @@ -181,10 +177,14 @@ async def collect_stream_docs( self._emitted_resource = bundler_composer( mimetype=self._mimetype, uri=uri, + # TODO: This is confusing, I expected this to be of type `DataKey` + # but it is a string. Naming could be improved maybe? data_key=self._name_provider(), + # Q: What are the parameters used for? Extra info? parameters={ - # Assume that we always write 1 frame per file/chunk - "chunk_shape": (1, *frame_shape), + # TODO: Validate this assumption and that it should not be self._batch_size + # Assume that we always write self._batch_size frames per file/chunk + "chunk_shape": (self._batch_size, *frame_shape), # Include file template for reconstruction in consolidator "template": file_template, }, diff --git a/src/ophyd_async/epics/eiger/_odin_io.py b/src/ophyd_async/epics/eiger/_odin_io.py index c051fe3549..4d99d5adae 100644 --- a/src/ophyd_async/epics/eiger/_odin_io.py +++ b/src/ophyd_async/epics/eiger/_odin_io.py @@ -2,7 +2,7 @@ from collections.abc import AsyncGenerator, AsyncIterator from bluesky.protocols import StreamAsset -from event_model import DataKey +from event_model import DataKey # type: ignore from ophyd_async.core import ( DEFAULT_TIMEOUT, @@ -114,10 +114,10 @@ async def observe_indices_written( self, timeout=DEFAULT_TIMEOUT ) -> AsyncGenerator[int, None]: async for num_captured in observe_value(self._drv.num_captured, timeout): - yield num_captured + yield num_captured // self._batch_size async def get_indices_written(self) -> int: - return await self._drv.num_captured.get_value() + return await self._drv.num_captured.get_value() // self._batch_size def collect_stream_docs(self, indices_written: int) -> AsyncIterator[StreamAsset]: # TODO: Correctly return stream https://github.com/bluesky/ophyd-async/issues/530 diff --git a/src/ophyd_async/fastcs/panda/_writer.py b/src/ophyd_async/fastcs/panda/_writer.py index 4037582089..68bdd4f800 100644 --- a/src/ophyd_async/fastcs/panda/_writer.py +++ b/src/ophyd_async/fastcs/panda/_writer.py @@ -34,7 +34,6 @@ def __init__( self._name_provider = name_provider self._datasets: list[HDFDataset] = [] self._file: HDFFile | None = None - self._batch_size = 1 # Triggered on PCAP arm async def open(self, batch_size: int = 1) -> dict[str, DataKey]: @@ -84,7 +83,8 @@ async def _describe(self) -> dict[str, DataKey]: describe = { ds.data_key: DataKey( source=self.panda_data_block.hdf_directory.source, - shape=list((self._batch_size, *ds.shape)), + # batch_size is always 1 for PandA + shape=list((1, *ds.shape)), dtype="array" if ds.shape != [1] else "number", # PandA data should always be written as Float64 dtype_numpy=" dict[str, DataKey]: + self._batch_size = batch_size return await self.pattern_generator.open_file( self.path_provider, self.name_provider(), batch_size ) @@ -35,7 +37,7 @@ async def observe_indices_written( self, timeout=DEFAULT_TIMEOUT ) -> AsyncGenerator[int, None]: async for index in self.pattern_generator.observe_indices_written(timeout): - yield index + yield index // self._batch_size async def get_indices_written(self) -> int: - return self.pattern_generator.image_counter + return self.pattern_generator.image_counter // self._batch_size diff --git a/tests/core/test_flyer.py b/tests/core/test_flyer.py index f9931b5e50..27271376c6 100644 --- a/tests/core/test_flyer.py +++ b/tests/core/test_flyer.py @@ -9,7 +9,7 @@ import pytest from bluesky.protocols import StreamAsset from bluesky.run_engine import RunEngine -from event_model import ComposeStreamResourceBundle, DataKey, compose_stream_resource +from event_model import ComposeStreamResourceBundle, DataKey, compose_stream_resource # type: ignore from pydantic import ValidationError from ophyd_async.core import ( @@ -54,13 +54,13 @@ async def stop(self): class DummyWriter(DetectorWriter): - def __init__(self, name: str, shape: Sequence[int]): + def __init__(self, name: str, shape: Sequence[int], batch_size: int = 1): self.dummy_signal = epics_signal_rw(int, "pva://read_pv") self._shape = shape self._name = name + self._batch_size = batch_size self._file: ComposeStreamResourceBundle | None = None self._last_emitted = 0 - self._batch_size = 1 self.index = 0 async def open(self, batch_size: int = 1) -> dict[str, DataKey]: diff --git a/tests/plan_stubs/test_fly.py b/tests/plan_stubs/test_fly.py index f20f052b52..eb19944683 100644 --- a/tests/plan_stubs/test_fly.py +++ b/tests/plan_stubs/test_fly.py @@ -6,7 +6,7 @@ import pytest from bluesky.protocols import StreamAsset from bluesky.run_engine import RunEngine -from event_model import ComposeStreamResourceBundle, DataKey, compose_stream_resource +from event_model import ComposeStreamResourceBundle, DataKey, compose_stream_resource # type: ignore from ophyd_async.core import ( DEFAULT_TIMEOUT, @@ -38,13 +38,13 @@ class DummyWriter(DetectorWriter): - def __init__(self, name: str, shape: Sequence[int]): + def __init__(self, name: str, shape: Sequence[int], batch_size: int = 1): self.dummy_signal = epics_signal_rw(int, "pva://read_pv") self._shape = shape self._name = name + self._batch_size = batch_size self._file: ComposeStreamResourceBundle | None = None self._last_emitted = 0 - self._batch_size = 1 self.index = 0 self.observe_indices_written_timeout_log = [] From 753eee807e47cc3fb20ba2392a866c606a479387 Mon Sep 17 00:00:00 2001 From: thomashopkins32 Date: Fri, 27 Dec 2024 09:11:53 -0500 Subject: [PATCH 43/60] Get the initial frame index after opening the writer --- src/ophyd_async/core/_detector.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/ophyd_async/core/_detector.py b/src/ophyd_async/core/_detector.py index 5ca10bf9e5..8b14372cbc 100644 --- a/src/ophyd_async/core/_detector.py +++ b/src/ophyd_async/core/_detector.py @@ -150,7 +150,7 @@ async def open(self, batch_size: int = 1) -> dict[str, DataKey]: def observe_indices_written( self, timeout=DEFAULT_TIMEOUT ) -> AsyncGenerator[int, None]: - """Yield the index of each frame (or equivalent data point) as it is written""" + """Yield the index of each frame (or batch of frames) as it is written""" @abstractmethod async def get_indices_written(self) -> int: @@ -329,13 +329,12 @@ async def prepare(self, value: TriggerInfo) -> None: if isinstance(self._trigger_info.number_of_triggers, list) else [self._trigger_info.number_of_triggers] ) - # TODO: How can we get the indices written prior to opening the writer? - # This is a problem since the `get_indices_written` needs to know the batch_size - # to return the correct number of indices written. - self._initial_frame = await self._writer.get_indices_written() + # Open the writer and prepare the controller. self._describe, _ = await asyncio.gather( self._writer.open(value.batch_size), self._controller.prepare(value) ) + # Get the initial frame index from the writer. + self._initial_frame = await self._writer.get_indices_written() if value.trigger != DetectorTrigger.INTERNAL: await self._controller.arm() self._fly_start = time.monotonic() From 26387e5be3296455e107fbbd9dcb57d775ac7bca Mon Sep 17 00:00:00 2001 From: thomashopkins32 Date: Fri, 27 Dec 2024 09:20:33 -0500 Subject: [PATCH 44/60] Remove batch_size from any writer __init__ --- src/ophyd_async/epics/adcore/_core_writer.py | 1 - .../sim/_pattern_detector/_pattern_detector_writer.py | 1 - tests/core/test_flyer.py | 3 +-- tests/plan_stubs/test_fly.py | 3 +-- 4 files changed, 2 insertions(+), 6 deletions(-) diff --git a/src/ophyd_async/epics/adcore/_core_writer.py b/src/ophyd_async/epics/adcore/_core_writer.py index 5cb08fa0a1..40c002f1d5 100644 --- a/src/ophyd_async/epics/adcore/_core_writer.py +++ b/src/ophyd_async/epics/adcore/_core_writer.py @@ -55,7 +55,6 @@ def __init__( self._emitted_resource = None self._capture_status: AsyncStatus | None = None - self._batch_size = 1 self._filename_template = "%s%s_%6.6d" @classmethod diff --git a/src/ophyd_async/sim/_pattern_detector/_pattern_detector_writer.py b/src/ophyd_async/sim/_pattern_detector/_pattern_detector_writer.py index 1f6a634b9d..72f50492ca 100644 --- a/src/ophyd_async/sim/_pattern_detector/_pattern_detector_writer.py +++ b/src/ophyd_async/sim/_pattern_detector/_pattern_detector_writer.py @@ -19,7 +19,6 @@ def __init__( self.pattern_generator = pattern_generator self.path_provider = path_provider self.name_provider = name_provider - self._batch_size = 1 async def open(self, batch_size: int = 1) -> dict[str, DataKey]: self._batch_size = batch_size diff --git a/tests/core/test_flyer.py b/tests/core/test_flyer.py index 27271376c6..ddb17ffeea 100644 --- a/tests/core/test_flyer.py +++ b/tests/core/test_flyer.py @@ -54,11 +54,10 @@ async def stop(self): class DummyWriter(DetectorWriter): - def __init__(self, name: str, shape: Sequence[int], batch_size: int = 1): + def __init__(self, name: str, shape: Sequence[int]): self.dummy_signal = epics_signal_rw(int, "pva://read_pv") self._shape = shape self._name = name - self._batch_size = batch_size self._file: ComposeStreamResourceBundle | None = None self._last_emitted = 0 self.index = 0 diff --git a/tests/plan_stubs/test_fly.py b/tests/plan_stubs/test_fly.py index eb19944683..bad9db0c03 100644 --- a/tests/plan_stubs/test_fly.py +++ b/tests/plan_stubs/test_fly.py @@ -38,11 +38,10 @@ class DummyWriter(DetectorWriter): - def __init__(self, name: str, shape: Sequence[int], batch_size: int = 1): + def __init__(self, name: str, shape: Sequence[int]): self.dummy_signal = epics_signal_rw(int, "pva://read_pv") self._shape = shape self._name = name - self._batch_size = batch_size self._file: ComposeStreamResourceBundle | None = None self._last_emitted = 0 self.index = 0 From f6a4f307b4c99f7e52ac57c529230f410cc574b5 Mon Sep 17 00:00:00 2001 From: thomashopkins32 Date: Fri, 27 Dec 2024 10:05:44 -0500 Subject: [PATCH 45/60] Call tiff_writer.open before collect_stream_docs --- src/ophyd_async/epics/adcore/_core_writer.py | 7 ++++++- tests/epics/adcore/test_writers.py | 1 + 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/ophyd_async/epics/adcore/_core_writer.py b/src/ophyd_async/epics/adcore/_core_writer.py index 40c002f1d5..8ee645983f 100644 --- a/src/ophyd_async/epics/adcore/_core_writer.py +++ b/src/ophyd_async/epics/adcore/_core_writer.py @@ -5,7 +5,11 @@ from urllib.parse import urlunparse from bluesky.protocols import Hints, StreamAsset -from event_model import DataKey, ComposeStreamResource, StreamRange # type: ignore +from event_model import ( # type: ignore + ComposeStreamResource, + DataKey, + StreamRange, +) from ophyd_async.core._detector import DetectorWriter from ophyd_async.core._providers import DatasetDescriber, NameProvider, PathProvider @@ -91,6 +95,7 @@ async def begin_capture(self) -> None: # Set the directory creation depth first, since dir creation callback happens # when directory path PV is processed. + # TODO: This is not actually creating the directory, it's just setting the depth await self.fileio.create_directory.set(info.create_dir_depth) await asyncio.gather( diff --git a/tests/epics/adcore/test_writers.py b/tests/epics/adcore/test_writers.py index 6e5fa46441..8b82ae7d63 100644 --- a/tests/epics/adcore/test_writers.py +++ b/tests/epics/adcore/test_writers.py @@ -99,6 +99,7 @@ async def test_hdf_writer_collect_stream_docs(hdf_writer: adcore.ADHDFWriter): async def test_tiff_writer_collect_stream_docs(tiff_writer: adcore.ADTIFFWriter): assert tiff_writer._emitted_resource is None + _ = await tiff_writer.open(batch_size=1) [item async for item in tiff_writer.collect_stream_docs(1)] assert tiff_writer._emitted_resource From dcc15660f161fc0dd7836f034e3f6425ed1d846f Mon Sep 17 00:00:00 2001 From: thomashopkins32 Date: Mon, 6 Jan 2025 10:35:25 -0500 Subject: [PATCH 46/60] Fixed tests by adding mocked directory creation callback to tiff and hdf writers --- src/ophyd_async/epics/adcore/_core_writer.py | 1 - tests/conftest.py | 2 +- tests/epics/adcore/test_writers.py | 48 ++++++++++++++++++-- tests/epics/adpilatus/test_pilatus.py | 4 -- 4 files changed, 44 insertions(+), 11 deletions(-) diff --git a/src/ophyd_async/epics/adcore/_core_writer.py b/src/ophyd_async/epics/adcore/_core_writer.py index 8ee645983f..d27f6d3935 100644 --- a/src/ophyd_async/epics/adcore/_core_writer.py +++ b/src/ophyd_async/epics/adcore/_core_writer.py @@ -95,7 +95,6 @@ async def begin_capture(self) -> None: # Set the directory creation depth first, since dir creation callback happens # when directory path PV is processed. - # TODO: This is not actually creating the directory, it's just setting the depth await self.fileio.create_directory.set(info.create_dir_depth) await asyncio.gather( diff --git a/tests/conftest.py b/tests/conftest.py index cfda52b3b1..69d4cfdf4f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -270,7 +270,7 @@ async def sim_detector(request: FixtureRequest): async with init_devices(mock=True): det = adsimdetector.SimDetector(prefix, dp, name=name) - det._config_sigs = [det.drv.acquire_time, det.drv.acquire] + det._config_sigs = [det.driver.acquire_time, det.driver.acquire] return det diff --git a/tests/epics/adcore/test_writers.py b/tests/epics/adcore/test_writers.py index 8b82ae7d63..3f54b895f9 100644 --- a/tests/epics/adcore/test_writers.py +++ b/tests/epics/adcore/test_writers.py @@ -1,3 +1,5 @@ +import os + import xml.etree.ElementTree as ET from unittest.mock import patch @@ -14,7 +16,7 @@ from ophyd_async.epics.adpilatus import PilatusReadoutTime from ophyd_async.epics.core import epics_signal_r from ophyd_async.plan_stubs import setup_ndattributes, setup_ndstats_sum -from ophyd_async.testing import set_mock_value +from ophyd_async.testing import set_mock_value, callback_on_mock_put class DummyDatasetDescriber(DatasetDescriber): @@ -32,7 +34,7 @@ async def hdf_writer( async with init_devices(mock=True): hdf = adcore.NDFileHDFIO("HDF:") - return adcore.ADHDFWriter( + writer = adcore.ADHDFWriter( hdf, static_path_provider, lambda: "test", @@ -40,6 +42,18 @@ async def hdf_writer( {}, ) + def on_set_file_path_callback(value: str, wait: bool = True): + """Mock a successful directory & file creation""" + set_mock_value(writer.fileio.file_path_exists, True) + set_mock_value( + writer.fileio.full_file_name, + f"{value}/{static_path_provider._filename_provider()}{writer._file_extension}", + ) + + callback_on_mock_put(writer.fileio.file_path, on_set_file_path_callback) + + return writer + @pytest.fixture async def tiff_writer( @@ -48,10 +62,22 @@ async def tiff_writer( async with init_devices(mock=True): tiff = adcore.NDFileIO("TIFF:") - return adcore.ADTIFFWriter( + writer = adcore.ADTIFFWriter( tiff, static_path_provider, lambda: "test", DummyDatasetDescriber(), {} ) + def on_set_file_path_callback(value: str, wait: bool = True): + """Mock a successful directory & file creation""" + set_mock_value(writer.fileio.file_path_exists, True) + set_mock_value( + writer.fileio.full_file_name, + f"{value}/{static_path_provider._filename_provider()}{writer._file_extension}", + ) + + callback_on_mock_put(writer.fileio.file_path, on_set_file_path_callback) + + return writer + @pytest.fixture async def hdf_writer_with_stats( @@ -64,7 +90,7 @@ async def hdf_writer_with_stats( # Set number of frames per chunk to something reasonable set_mock_value(hdf.num_frames_chunks, 2) - return adcore.ADHDFWriter( + writer = adcore.ADHDFWriter( hdf, static_path_provider, lambda: "test", @@ -72,6 +98,18 @@ async def hdf_writer_with_stats( {"stats": stats}, ) + def on_set_file_path_callback(value: str, wait: bool = True): + """Mock a successful directory & file creation""" + set_mock_value(writer.fileio.file_path_exists, True) + set_mock_value( + writer.fileio.full_file_name, + f"{value}/{static_path_provider._filename_provider()}{writer._file_extension}", + ) + + callback_on_mock_put(writer.fileio.file_path, on_set_file_path_callback) + + return writer + @pytest.fixture async def detectors( @@ -92,7 +130,7 @@ async def detectors( async def test_hdf_writer_collect_stream_docs(hdf_writer: adcore.ADHDFWriter): assert hdf_writer._file is None - + _ = await hdf_writer.open(batch_size=1) [item async for item in hdf_writer.collect_stream_docs(1)] assert hdf_writer._file diff --git a/tests/epics/adpilatus/test_pilatus.py b/tests/epics/adpilatus/test_pilatus.py index fce84c7d92..bcdd2df6ba 100644 --- a/tests/epics/adpilatus/test_pilatus.py +++ b/tests/epics/adpilatus/test_pilatus.py @@ -121,10 +121,6 @@ async def test_unsupported_trigger_excepts(test_adpilatus: adpilatus.PilatusDete async def test_exposure_time_and_acquire_period_set( test_adpilatus: adpilatus.PilatusDetector, ): - async def dummy_open(batch_size: int = 0): - return {} - - test_adpilatus._writer.open = dummy_open set_mock_value(test_adpilatus.driver.armed, True) await test_adpilatus.prepare( TriggerInfo( From 2bd6186ccac6a7c6fa38162db32c6177ef6b8458 Mon Sep 17 00:00:00 2001 From: thomashopkins32 Date: Wed, 8 Jan 2025 09:41:43 -0500 Subject: [PATCH 47/60] Change batch_size -> frames_per_event --- src/ophyd_async/core/_detector.py | 18 ++++++++-------- src/ophyd_async/core/_hdf_dataset.py | 4 ++-- .../epics/adaravis/_aravis_controller.py | 4 +--- src/ophyd_async/epics/adcore/_core_writer.py | 16 +++++++------- src/ophyd_async/epics/adcore/_hdf_writer.py | 10 ++++----- src/ophyd_async/epics/eiger/_odin_io.py | 10 ++++----- src/ophyd_async/fastcs/panda/_writer.py | 10 ++++----- .../_pattern_detector_writer.py | 10 ++++----- .../_pattern_detector/_pattern_generator.py | 12 +++++------ tests/core/test_flyer.py | 12 +++++------ tests/epics/adaravis/test_aravis.py | 2 +- tests/epics/adcore/test_writers.py | 21 ++----------------- tests/epics/adkinetix/test_kinetix.py | 1 - tests/epics/advimba/test_vimba.py | 1 - tests/fastcs/panda/test_hdf_panda.py | 3 +-- tests/fastcs/panda/test_writer.py | 5 ++--- tests/plan_stubs/test_fly.py | 11 +++++----- 17 files changed, 63 insertions(+), 87 deletions(-) diff --git a/src/ophyd_async/core/_detector.py b/src/ophyd_async/core/_detector.py index 8b14372cbc..110d7549d1 100644 --- a/src/ophyd_async/core/_detector.py +++ b/src/ophyd_async/core/_detector.py @@ -66,10 +66,10 @@ class TriggerInfo(BaseModel): #: What is the maximum timeout on waiting for a frame frame_timeout: float | None = Field(default=None, gt=0) #: The number of triggers that are grouped into a single StreamDatum index. - #: A batch_size > 1 can be useful to have frames from a faster detector able to be zipped with a single frame from a slower detector. - #: E.g. if number_of_triggers=10 and batch_size=5 then the detector will take 10 frames, + #: A frames_per_event > 1 can be useful to have frames from a faster detector able to be zipped with a single frame from a slower detector. + #: E.g. if number_of_triggers=10 and frames_per_event=5 then the detector will take 10 frames, #: but publish 2 StreamDatum indices, and describe() will show a shape of (5, h, w) for each. - batch_size: NonNegativeInt = 1 + frames_per_event: NonNegativeInt = 1 @computed_field @cached_property @@ -107,7 +107,7 @@ async def prepare(self, trigger_info: TriggerInfo) -> None: exposure time. deadtime Defaults to None. This is the minimum deadtime between triggers. - batch_size The number of triggers grouped into a single StreamDatum + frames_per_event The number of triggers grouped into a single StreamDatum index. """ @@ -133,13 +133,13 @@ class DetectorWriter(ABC): (e.g. an HDF5 file)""" @abstractmethod - async def open(self, batch_size: int = 1) -> dict[str, DataKey]: + async def open(self, frames_per_event: int = 1) -> dict[str, DataKey]: """Open writer and wait for it to be ready for data. Args: - batch_size: The number of triggers are grouped into a single StreamDatum index. - A batch_size > 1 can be useful to have frames from a faster detector able to be zipped with a single frame from a slow detector. - E.g. if number_of_triggers=10 and batch_size=5 then the detector will take 10 frames, + frames_per_event: The number of triggers are grouped into a single StreamDatum index. + A frames_per_event > 1 can be useful to have frames from a faster detector able to be zipped with a single frame from a slow detector. + E.g. if number_of_triggers=10 and frames_per_event=5 then the detector will take 10 frames, but publish 2 StreamDatum indices, and describe() will show a shape of (5, h, w) for each. Returns: @@ -331,7 +331,7 @@ async def prepare(self, value: TriggerInfo) -> None: ) # Open the writer and prepare the controller. self._describe, _ = await asyncio.gather( - self._writer.open(value.batch_size), self._controller.prepare(value) + self._writer.open(value.frames_per_event), self._controller.prepare(value) ) # Get the initial frame index from the writer. self._initial_frame = await self._writer.get_indices_written() diff --git a/src/ophyd_async/core/_hdf_dataset.py b/src/ophyd_async/core/_hdf_dataset.py index ab08a09177..ec8fe7b8fe 100644 --- a/src/ophyd_async/core/_hdf_dataset.py +++ b/src/ophyd_async/core/_hdf_dataset.py @@ -18,7 +18,7 @@ class HDFDataset: dataset: str shape: Sequence[int] = field(default_factory=tuple) dtype_numpy: str = "" - batch_size: int = 1 + frames_per_event: int = 1 swmr: bool = False # Represents explicit chunk size written to disk. chunk_shape: tuple[int, ...] = () @@ -67,7 +67,7 @@ def __init__( parameters={ "dataset": ds.dataset, "swmr": ds.swmr, - "batch_size": ds.batch_size, + "frames_per_event": ds.frames_per_event, "chunk_shape": ds.chunk_shape, }, uid=None, diff --git a/src/ophyd_async/epics/adaravis/_aravis_controller.py b/src/ophyd_async/epics/adaravis/_aravis_controller.py index 88763fe988..7fbc21c88f 100644 --- a/src/ophyd_async/epics/adaravis/_aravis_controller.py +++ b/src/ophyd_async/epics/adaravis/_aravis_controller.py @@ -1,5 +1,5 @@ import asyncio -from typing import Literal, TypeVar +from typing import Literal from ophyd_async.core import ( DetectorTrigger, @@ -14,8 +14,6 @@ # runtime. See https://github.com/bluesky/ophyd-async/issues/308 _HIGHEST_POSSIBLE_DEADTIME = 1961e-6 -AravisControllerT = TypeVar("AravisControllerT", bound="AravisController") - class AravisController(adcore.ADBaseController[AravisDriverIO]): GPIO_NUMBER = Literal[1, 2, 3, 4] diff --git a/src/ophyd_async/epics/adcore/_core_writer.py b/src/ophyd_async/epics/adcore/_core_writer.py index d27f6d3935..b91c3673fa 100644 --- a/src/ophyd_async/epics/adcore/_core_writer.py +++ b/src/ophyd_async/epics/adcore/_core_writer.py @@ -122,10 +122,10 @@ async def begin_capture(self) -> None: self.fileio.capture, True, wait_for_set_completion=False ) - async def open(self, batch_size: int = 1) -> dict[str, DataKey]: + async def open(self, frames_per_event: int = 1) -> dict[str, DataKey]: self._emitted_resource = None self._last_emitted = 0 - self._batch_size = batch_size + self._frames_per_event = frames_per_event frame_shape = await self._dataset_describer.shape() dtype_numpy = await self._dataset_describer.np_datatype() @@ -134,7 +134,7 @@ async def open(self, batch_size: int = 1) -> dict[str, DataKey]: describe = { self._name_provider(): DataKey( source=self._name_provider(), - shape=list((batch_size, *frame_shape)), + shape=list((frames_per_event, *frame_shape)), dtype="array", dtype_numpy=dtype_numpy, external="STREAM:", @@ -147,11 +147,11 @@ async def observe_indices_written( ) -> AsyncGenerator[int, None]: """Wait until a specific index is ready to be collected""" async for num_captured in observe_value(self.fileio.num_captured, timeout): - yield num_captured // self._batch_size + yield num_captured // self._frames_per_event async def get_indices_written(self) -> int: num_captured = await self.fileio.num_captured.get_value() - return num_captured // self._batch_size + return num_captured // self._frames_per_event async def collect_stream_docs( self, indices_written: int @@ -185,9 +185,9 @@ async def collect_stream_docs( data_key=self._name_provider(), # Q: What are the parameters used for? Extra info? parameters={ - # TODO: Validate this assumption and that it should not be self._batch_size - # Assume that we always write self._batch_size frames per file/chunk - "chunk_shape": (self._batch_size, *frame_shape), + # TODO: Validate this assumption and that it should not be self._frames_per_event + # Assume that we always write self._frames_per_event frames per file/chunk + "chunk_shape": (self._frames_per_event, *frame_shape), # Include file template for reconstruction in consolidator "template": file_template, }, diff --git a/src/ophyd_async/epics/adcore/_hdf_writer.py b/src/ophyd_async/epics/adcore/_hdf_writer.py index 91ff97ec55..7e91c6fb18 100644 --- a/src/ophyd_async/epics/adcore/_hdf_writer.py +++ b/src/ophyd_async/epics/adcore/_hdf_writer.py @@ -48,7 +48,7 @@ def __init__( self._file: HDFFile | None = None self._include_file_number = False - async def open(self, batch_size: int = 1) -> dict[str, DataKey]: + async def open(self, frames_per_event: int = 1) -> dict[str, DataKey]: self._file = None # Setting HDF writer specific signals @@ -74,7 +74,7 @@ async def open(self, batch_size: int = 1) -> dict[str, DataKey]: name = self._name_provider() detector_shape = await self._dataset_describer.shape() np_dtype = await self._dataset_describer.np_datatype() - self._batch_size = batch_size + self._frames_per_event = frames_per_event # Determine number of frames that will be saved per HDF chunk frames_per_chunk = await self.fileio.num_frames_chunks.get_value() @@ -86,7 +86,7 @@ async def open(self, batch_size: int = 1) -> dict[str, DataKey]: dataset="/entry/data/data", shape=detector_shape, dtype_numpy=np_dtype, - batch_size=batch_size, + frames_per_event=frames_per_event, chunk_shape=(frames_per_chunk, *detector_shape), ) ] @@ -113,7 +113,7 @@ async def open(self, batch_size: int = 1) -> dict[str, DataKey]: f"/entry/instrument/NDAttributes/{datakey}", (), np_datatype, - batch_size, + frames_per_event, # NDAttributes appear to always be configured with # this chunk size chunk_shape=(16384,), @@ -123,7 +123,7 @@ async def open(self, batch_size: int = 1) -> dict[str, DataKey]: describe = { ds.data_key: DataKey( source=self.fileio.full_file_name.source, - shape=list((batch_size, *ds.shape)), + shape=list((frames_per_event, *ds.shape)), dtype="array" if ds.shape else "number", dtype_numpy=ds.dtype_numpy, external="STREAM:", diff --git a/src/ophyd_async/epics/eiger/_odin_io.py b/src/ophyd_async/epics/eiger/_odin_io.py index 4d99d5adae..1922b7f528 100644 --- a/src/ophyd_async/epics/eiger/_odin_io.py +++ b/src/ophyd_async/epics/eiger/_odin_io.py @@ -77,9 +77,9 @@ def __init__( self._name_provider = name_provider super().__init__() - async def open(self, batch_size: int = 1) -> dict[str, DataKey]: + async def open(self, frames_per_event: int = 1) -> dict[str, DataKey]: info = self._path_provider(device_name=self._name_provider()) - self._batch_size = batch_size + self._frames_per_event = frames_per_event await asyncio.gather( self._drv.file_path.set(str(info.directory_path)), @@ -102,7 +102,7 @@ async def _describe(self) -> dict[str, DataKey]: return { "data": DataKey( source=self._drv.file_name.source, - shape=list((self._batch_size, *data_shape)), + shape=list((self._frames_per_event, *data_shape)), dtype="array", # TODO: Use correct type based on eiger https://github.com/bluesky/ophyd-async/issues/529 dtype_numpy=" AsyncGenerator[int, None]: async for num_captured in observe_value(self._drv.num_captured, timeout): - yield num_captured // self._batch_size + yield num_captured // self._frames_per_event async def get_indices_written(self) -> int: - return await self._drv.num_captured.get_value() // self._batch_size + return await self._drv.num_captured.get_value() // self._frames_per_event def collect_stream_docs(self, indices_written: int) -> AsyncIterator[StreamAsset]: # TODO: Correctly return stream https://github.com/bluesky/ophyd-async/issues/530 diff --git a/src/ophyd_async/fastcs/panda/_writer.py b/src/ophyd_async/fastcs/panda/_writer.py index 68bdd4f800..3f4243482f 100644 --- a/src/ophyd_async/fastcs/panda/_writer.py +++ b/src/ophyd_async/fastcs/panda/_writer.py @@ -36,7 +36,7 @@ def __init__( self._file: HDFFile | None = None # Triggered on PCAP arm - async def open(self, batch_size: int = 1) -> dict[str, DataKey]: + async def open(self, frames_per_event: int = 1) -> dict[str, DataKey]: """Retrieve and get descriptor of all PandA signals marked for capture""" # Ensure flushes are immediate @@ -67,9 +67,9 @@ async def open(self, batch_size: int = 1) -> dict[str, DataKey]: # Wait for it to start, stashing the status that tells us when it finishes await self.panda_data_block.capture.set(True) - if batch_size > 1: + if frames_per_event > 1: raise ValueError( - "All PandA datasets should be scalar, batch_size should be 1" + "All PandA datasets should be scalar, frames_per_event should be 1" ) return await self._describe() @@ -83,7 +83,7 @@ async def _describe(self) -> dict[str, DataKey]: describe = { ds.data_key: DataKey( source=self.panda_data_block.hdf_directory.source, - # batch_size is always 1 for PandA + # frames_per_event is always 1 for PandA shape=list((1, *ds.shape)), dtype="array" if ds.shape != [1] else "number", # PandA data should always be written as Float64 @@ -105,7 +105,7 @@ async def _update_datasets(self) -> None: # TODO: Update chunk size to read signal once available in IOC # Currently PandA IOC sets chunk size to 1024 points per chunk HDFDataset( - dataset_name, "/" + dataset_name, [1], batch_size=1, chunk_shape=(1024,) + dataset_name, "/" + dataset_name, [1], frames_per_event=1, chunk_shape=(1024,) ) for dataset_name in capture_table.name ] diff --git a/src/ophyd_async/sim/_pattern_detector/_pattern_detector_writer.py b/src/ophyd_async/sim/_pattern_detector/_pattern_detector_writer.py index 72f50492ca..2513b39559 100644 --- a/src/ophyd_async/sim/_pattern_detector/_pattern_detector_writer.py +++ b/src/ophyd_async/sim/_pattern_detector/_pattern_detector_writer.py @@ -20,10 +20,10 @@ def __init__( self.path_provider = path_provider self.name_provider = name_provider - async def open(self, batch_size: int = 1) -> dict[str, DataKey]: - self._batch_size = batch_size + async def open(self, frames_per_event: int = 1) -> dict[str, DataKey]: + self._frames_per_event = frames_per_event return await self.pattern_generator.open_file( - self.path_provider, self.name_provider(), batch_size + self.path_provider, self.name_provider(), frames_per_event ) async def close(self) -> None: @@ -36,7 +36,7 @@ async def observe_indices_written( self, timeout=DEFAULT_TIMEOUT ) -> AsyncGenerator[int, None]: async for index in self.pattern_generator.observe_indices_written(timeout): - yield index // self._batch_size + yield index // self._frames_per_event async def get_indices_written(self) -> int: - return self.pattern_generator.image_counter // self._batch_size + return self.pattern_generator.image_counter // self._frames_per_event diff --git a/src/ophyd_async/sim/_pattern_detector/_pattern_generator.py b/src/ophyd_async/sim/_pattern_detector/_pattern_generator.py index 5c9e3f4cb8..16a89fcaf4 100644 --- a/src/ophyd_async/sim/_pattern_detector/_pattern_generator.py +++ b/src/ophyd_async/sim/_pattern_detector/_pattern_generator.py @@ -105,7 +105,7 @@ def set_y(self, value: float) -> None: self.y = value async def open_file( - self, path_provider: PathProvider, name: str, batch_size: int = 1 + self, path_provider: PathProvider, name: str, frames_per_event: int = 1 ) -> dict[str, DataKey]: await self.counter_signal.connect() @@ -131,7 +131,7 @@ async def open_file( # once datasets written, can switch the model to single writer multiple reader self._handle_for_h5_file.swmr_mode = True - self._batch_size = batch_size + self._frames_per_event = frames_per_event # cache state to self # Add the main data @@ -140,20 +140,20 @@ async def open_file( data_key=name, dataset=DATA_PATH, shape=(self.height, self.width), - batch_size=batch_size, + frames_per_event=frames_per_event, ), HDFDataset( f"{name}-sum", dataset=SUM_PATH, shape=(), - batch_size=batch_size, + frames_per_event=frames_per_event, ), ] describe = { ds.data_key: DataKey( source="sim://pattern-generator-hdf-file", - shape=list((self._batch_size, *ds.shape)), + shape=list((self._frames_per_event, *ds.shape)), dtype="array" if ds.shape else "number", external="STREAM:", ) @@ -202,4 +202,4 @@ async def observe_indices_written( self, timeout=DEFAULT_TIMEOUT ) -> AsyncGenerator[int, None]: async for num_captured in observe_value(self.counter_signal, timeout=timeout): - yield num_captured // self._batch_size + yield num_captured // self._frames_per_event diff --git a/tests/core/test_flyer.py b/tests/core/test_flyer.py index ddb17ffeea..9b43072556 100644 --- a/tests/core/test_flyer.py +++ b/tests/core/test_flyer.py @@ -62,12 +62,12 @@ def __init__(self, name: str, shape: Sequence[int]): self._last_emitted = 0 self.index = 0 - async def open(self, batch_size: int = 1) -> dict[str, DataKey]: - self._batch_size = batch_size + async def open(self, frames_per_event: int = 1) -> dict[str, DataKey]: + self._frames_per_event = frames_per_event return { self._name: DataKey( source="soft://some-source", - shape=list((batch_size, *self._shape)), + shape=list((frames_per_event, *self._shape)), dtype="number", dtype_numpy=" AsyncGenerator[int, None]: num_captured: int async for num_captured in observe_value(self.dummy_signal, timeout): - yield num_captured // self._batch_size + yield num_captured // self._frames_per_event async def get_indices_written(self) -> int: - return self.index // self._batch_size + return self.index // self._frames_per_event async def collect_stream_docs( self, indices_written: int @@ -96,7 +96,7 @@ async def collect_stream_docs( parameters={ "path": "", "dataset": "", - "batch_size": self._batch_size, + "frames_per_event": self._frames_per_event, }, uid=None, validate=True, diff --git a/tests/epics/adaravis/test_aravis.py b/tests/epics/adaravis/test_aravis.py index aa8f097451..b4f1a49421 100644 --- a/tests/epics/adaravis/test_aravis.py +++ b/tests/epics/adaravis/test_aravis.py @@ -105,7 +105,7 @@ async def test_can_collect( assert stream_resource["parameters"] == { "dataset": "/entry/data/data", "swmr": False, - "batch_size": 1, + "frames_per_event": 1, "chunk_shape": (1, 10, 10), } assert docs[1][0] == "stream_datum" diff --git a/tests/epics/adcore/test_writers.py b/tests/epics/adcore/test_writers.py index 3036ca8e8a..259cd2f2e3 100644 --- a/tests/epics/adcore/test_writers.py +++ b/tests/epics/adcore/test_writers.py @@ -1,5 +1,3 @@ -import os - import xml.etree.ElementTree as ET from unittest.mock import patch @@ -40,21 +38,6 @@ async def hdf_writer( lambda: "test", DummyDatasetDescriber(), {}, -<<<<<<< HEAD -======= - ) - - -@pytest.fixture -async def tiff_writer( - RE, static_path_provider: StaticPathProvider -) -> adcore.ADTIFFWriter: - async with init_devices(mock=True): - tiff = adcore.NDFileIO("TIFF:") - - return adcore.ADTIFFWriter( - tiff, static_path_provider, lambda: "test", DummyDatasetDescriber(), {} ->>>>>>> 4f4458c916071859c44b03b2f39a6a9af05d4aac ) def on_set_file_path_callback(value: str, wait: bool = True): @@ -145,14 +128,14 @@ async def detectors( async def test_hdf_writer_collect_stream_docs(hdf_writer: adcore.ADHDFWriter): assert hdf_writer._file is None - _ = await hdf_writer.open(batch_size=1) + _ = await hdf_writer.open(frames_per_event=1) [item async for item in hdf_writer.collect_stream_docs(1)] assert hdf_writer._file async def test_tiff_writer_collect_stream_docs(tiff_writer: adcore.ADTIFFWriter): assert tiff_writer._emitted_resource is None - _ = await tiff_writer.open(batch_size=1) + _ = await tiff_writer.open(frames_per_event=1) [item async for item in tiff_writer.collect_stream_docs(1)] assert tiff_writer._emitted_resource diff --git a/tests/epics/adkinetix/test_kinetix.py b/tests/epics/adkinetix/test_kinetix.py index 20b7f8b2f6..c6db2da72e 100644 --- a/tests/epics/adkinetix/test_kinetix.py +++ b/tests/epics/adkinetix/test_kinetix.py @@ -100,7 +100,6 @@ async def test_can_collect( assert stream_resource["parameters"] == { "dataset": "/entry/data/data", "swmr": False, - "batch_size": 1, "chunk_shape": (1, 10, 10), } assert docs[1][0] == "stream_datum" diff --git a/tests/epics/advimba/test_vimba.py b/tests/epics/advimba/test_vimba.py index a619d443b7..89874ee978 100644 --- a/tests/epics/advimba/test_vimba.py +++ b/tests/epics/advimba/test_vimba.py @@ -118,7 +118,6 @@ async def test_can_collect( assert stream_resource["parameters"] == { "dataset": "/entry/data/data", "swmr": False, - "batch_size": 1, "chunk_shape": (1, 10, 10), } assert docs[1][0] == "stream_datum" diff --git a/tests/fastcs/panda/test_hdf_panda.py b/tests/fastcs/panda/test_hdf_panda.py index 5624639e03..6812a3cc36 100644 --- a/tests/fastcs/panda/test_hdf_panda.py +++ b/tests/fastcs/panda/test_hdf_panda.py @@ -146,8 +146,7 @@ def flying_plan(): "parameters": { "dataset": f"/{dataset_name}", "swmr": False, - "batch_size": 1, - "chunk_shape": (1024,), + "chunk_shape": (1, 1024), }, } assert "test-panda.h5" in stream_resource["uri"] diff --git a/tests/fastcs/panda/test_writer.py b/tests/fastcs/panda/test_writer.py index 32732315e7..62d218465c 100644 --- a/tests/fastcs/panda/test_writer.py +++ b/tests/fastcs/panda/test_writer.py @@ -149,7 +149,7 @@ async def test_open_sets_file_path_and_name(mock_writer: PandaHDFWriter, tmp_pat assert name == "data.h5" -async def test_open_errors_when_batch_size_not_one(mock_writer: PandaHDFWriter): +async def test_open_errors_when_frames_per_event_not_one(mock_writer: PandaHDFWriter): with pytest.raises(ValueError): await mock_writer.open(2) @@ -191,8 +191,7 @@ def assert_resource_document(name, resource_doc): "parameters": { "dataset": f"/{name}", "swmr": False, - "batch_size": 1, - "chunk_shape": (1024,), + "chunk_shape": (1, 1024), }, } assert os.path.join("mock_panda", "data.h5") in resource_doc["uri"] diff --git a/tests/plan_stubs/test_fly.py b/tests/plan_stubs/test_fly.py index bad9db0c03..1ec4843fb7 100644 --- a/tests/plan_stubs/test_fly.py +++ b/tests/plan_stubs/test_fly.py @@ -47,12 +47,12 @@ def __init__(self, name: str, shape: Sequence[int]): self.index = 0 self.observe_indices_written_timeout_log = [] - async def open(self, batch_size: int = 1) -> dict[str, DataKey]: - self._batch_size = batch_size + async def open(self, frames_per_event: int = 1) -> dict[str, DataKey]: + self._frames_per_event = frames_per_event return { self._name: DataKey( source="soft://some-source", - shape=list((batch_size, *self._shape)), + shape=list((frames_per_event, *self._shape)), dtype="number", external="STREAM:", ) @@ -64,10 +64,10 @@ async def observe_indices_written( self.observe_indices_written_timeout_log.append(timeout) num_captured: int async for num_captured in observe_value(self.dummy_signal, timeout): - yield num_captured // self._batch_size + yield num_captured // self._frames_per_event async def get_indices_written(self) -> int: - return self.index // self._batch_size + return self.index // self._frames_per_event async def collect_stream_docs( self, indices_written: int @@ -81,7 +81,6 @@ async def collect_stream_docs( parameters={ "path": "", "swmr": False, - "batch_size": 1, }, uid=None, validate=True, From df488971077626874faafacffbacd05eb3678e02 Mon Sep 17 00:00:00 2001 From: thomashopkins32 Date: Wed, 8 Jan 2025 10:47:11 -0500 Subject: [PATCH 48/60] Remove frames_per_event from HDFDataset, use as first dimension of shape instead --- src/ophyd_async/core/_hdf_dataset.py | 3 +-- src/ophyd_async/epics/adcore/_core_writer.py | 2 +- src/ophyd_async/epics/adcore/_hdf_writer.py | 10 +++++----- src/ophyd_async/fastcs/panda/_writer.py | 6 +++--- .../sim/_pattern_detector/_pattern_generator.py | 8 +++----- tests/core/test_flyer.py | 2 +- tests/epics/adaravis/test_aravis.py | 2 +- tests/epics/adcore/test_writers.py | 4 ++-- tests/epics/adkinetix/test_kinetix.py | 1 + tests/epics/advimba/test_vimba.py | 1 + tests/fastcs/panda/test_hdf_panda.py | 3 ++- tests/fastcs/panda/test_writer.py | 7 +++---- tests/sim/test_sim_writer.py | 2 +- 13 files changed, 25 insertions(+), 26 deletions(-) diff --git a/src/ophyd_async/core/_hdf_dataset.py b/src/ophyd_async/core/_hdf_dataset.py index ec8fe7b8fe..6e13c58e18 100644 --- a/src/ophyd_async/core/_hdf_dataset.py +++ b/src/ophyd_async/core/_hdf_dataset.py @@ -18,7 +18,6 @@ class HDFDataset: dataset: str shape: Sequence[int] = field(default_factory=tuple) dtype_numpy: str = "" - frames_per_event: int = 1 swmr: bool = False # Represents explicit chunk size written to disk. chunk_shape: tuple[int, ...] = () @@ -67,7 +66,7 @@ def __init__( parameters={ "dataset": ds.dataset, "swmr": ds.swmr, - "frames_per_event": ds.frames_per_event, + "shape": ds.shape, "chunk_shape": ds.chunk_shape, }, uid=None, diff --git a/src/ophyd_async/epics/adcore/_core_writer.py b/src/ophyd_async/epics/adcore/_core_writer.py index b91c3673fa..ff1a7938ab 100644 --- a/src/ophyd_async/epics/adcore/_core_writer.py +++ b/src/ophyd_async/epics/adcore/_core_writer.py @@ -134,7 +134,7 @@ async def open(self, frames_per_event: int = 1) -> dict[str, DataKey]: describe = { self._name_provider(): DataKey( source=self._name_provider(), - shape=list((frames_per_event, *frame_shape)), + shape=[frames_per_event, *frame_shape], dtype="array", dtype_numpy=dtype_numpy, external="STREAM:", diff --git a/src/ophyd_async/epics/adcore/_hdf_writer.py b/src/ophyd_async/epics/adcore/_hdf_writer.py index 7e91c6fb18..56b3dd4c0a 100644 --- a/src/ophyd_async/epics/adcore/_hdf_writer.py +++ b/src/ophyd_async/epics/adcore/_hdf_writer.py @@ -74,6 +74,8 @@ async def open(self, frames_per_event: int = 1) -> dict[str, DataKey]: name = self._name_provider() detector_shape = await self._dataset_describer.shape() np_dtype = await self._dataset_describer.np_datatype() + + # Used by the base class self._frames_per_event = frames_per_event # Determine number of frames that will be saved per HDF chunk @@ -84,9 +86,8 @@ async def open(self, frames_per_event: int = 1) -> dict[str, DataKey]: HDFDataset( data_key=name, dataset="/entry/data/data", - shape=detector_shape, + shape=(frames_per_event, *detector_shape) if frames_per_event > 1 or detector_shape else (), dtype_numpy=np_dtype, - frames_per_event=frames_per_event, chunk_shape=(frames_per_chunk, *detector_shape), ) ] @@ -111,9 +112,8 @@ async def open(self, frames_per_event: int = 1) -> dict[str, DataKey]: HDFDataset( datakey, f"/entry/instrument/NDAttributes/{datakey}", - (), + (frames_per_event,) if frames_per_event > 1 else (), np_datatype, - frames_per_event, # NDAttributes appear to always be configured with # this chunk size chunk_shape=(16384,), @@ -123,7 +123,7 @@ async def open(self, frames_per_event: int = 1) -> dict[str, DataKey]: describe = { ds.data_key: DataKey( source=self.fileio.full_file_name.source, - shape=list((frames_per_event, *ds.shape)), + shape=list(ds.shape), dtype="array" if ds.shape else "number", dtype_numpy=ds.dtype_numpy, external="STREAM:", diff --git a/src/ophyd_async/fastcs/panda/_writer.py b/src/ophyd_async/fastcs/panda/_writer.py index 3f4243482f..90df6bf51f 100644 --- a/src/ophyd_async/fastcs/panda/_writer.py +++ b/src/ophyd_async/fastcs/panda/_writer.py @@ -84,8 +84,8 @@ async def _describe(self) -> dict[str, DataKey]: ds.data_key: DataKey( source=self.panda_data_block.hdf_directory.source, # frames_per_event is always 1 for PandA - shape=list((1, *ds.shape)), - dtype="array" if ds.shape != [1] else "number", + shape=[1, *ds.shape] if ds.shape else [], + dtype="array" if ds.shape else "number", # PandA data should always be written as Float64 dtype_numpy=" None: # TODO: Update chunk size to read signal once available in IOC # Currently PandA IOC sets chunk size to 1024 points per chunk HDFDataset( - dataset_name, "/" + dataset_name, [1], frames_per_event=1, chunk_shape=(1024,) + dataset_name, "/" + dataset_name, shape=(), chunk_shape=(1024,) ) for dataset_name in capture_table.name ] diff --git a/src/ophyd_async/sim/_pattern_detector/_pattern_generator.py b/src/ophyd_async/sim/_pattern_detector/_pattern_generator.py index 16a89fcaf4..9a878e8c36 100644 --- a/src/ophyd_async/sim/_pattern_detector/_pattern_generator.py +++ b/src/ophyd_async/sim/_pattern_detector/_pattern_generator.py @@ -139,21 +139,19 @@ async def open_file( HDFDataset( data_key=name, dataset=DATA_PATH, - shape=(self.height, self.width), - frames_per_event=frames_per_event, + shape=(frames_per_event, self.height, self.width), ), HDFDataset( f"{name}-sum", dataset=SUM_PATH, - shape=(), - frames_per_event=frames_per_event, + shape=(frames_per_event,) if frames_per_event > 1 else (), ), ] describe = { ds.data_key: DataKey( source="sim://pattern-generator-hdf-file", - shape=list((self._frames_per_event, *ds.shape)), + shape=list(ds.shape), dtype="array" if ds.shape else "number", external="STREAM:", ) diff --git a/tests/core/test_flyer.py b/tests/core/test_flyer.py index 9b43072556..29c411ce7c 100644 --- a/tests/core/test_flyer.py +++ b/tests/core/test_flyer.py @@ -67,7 +67,7 @@ async def open(self, frames_per_event: int = 1) -> dict[str, DataKey]: return { self._name: DataKey( source="soft://some-source", - shape=list((frames_per_event, *self._shape)), + shape=[frames_per_event, *self._shape], dtype="number", dtype_numpy=" Date: Wed, 8 Jan 2025 13:01:59 -0500 Subject: [PATCH 49/60] Cleanup + ruff checks --- src/ophyd_async/core/_detector.py | 24 +++++++++++-------- .../epics/adaravis/_aravis_controller.py | 1 + src/ophyd_async/epics/adcore/_core_writer.py | 7 ++++-- src/ophyd_async/epics/adcore/_hdf_writer.py | 6 +++-- src/ophyd_async/epics/eiger/_odin_io.py | 4 ++-- src/ophyd_async/fastcs/panda/_writer.py | 4 +--- .../_pattern_detector_writer.py | 2 +- tests/core/test_flyer.py | 6 ++++- tests/epics/adcore/test_writers.py | 2 +- tests/plan_stubs/test_fly.py | 9 +++++-- 10 files changed, 41 insertions(+), 24 deletions(-) diff --git a/src/ophyd_async/core/_detector.py b/src/ophyd_async/core/_detector.py index 110d7549d1..b413cf7689 100644 --- a/src/ophyd_async/core/_detector.py +++ b/src/ophyd_async/core/_detector.py @@ -66,9 +66,11 @@ class TriggerInfo(BaseModel): #: What is the maximum timeout on waiting for a frame frame_timeout: float | None = Field(default=None, gt=0) #: The number of triggers that are grouped into a single StreamDatum index. - #: A frames_per_event > 1 can be useful to have frames from a faster detector able to be zipped with a single frame from a slower detector. - #: E.g. if number_of_triggers=10 and frames_per_event=5 then the detector will take 10 frames, - #: but publish 2 StreamDatum indices, and describe() will show a shape of (5, h, w) for each. + #: A frames_per_event > 1 can be useful to have frames from a faster detector + #: able to be zipped with a single frame from a slower detector. E.g. if + #: number_of_triggers=10 and frames_per_event=5 then the detector will take + #: 10 frames, but publish 2 StreamDatum indices, and describe() will show a + #: shape of (5, h, w) for each. frames_per_event: NonNegativeInt = 1 @computed_field @@ -107,8 +109,8 @@ async def prepare(self, trigger_info: TriggerInfo) -> None: exposure time. deadtime Defaults to None. This is the minimum deadtime between triggers. - frames_per_event The number of triggers grouped into a single StreamDatum - index. + frames_per_event The number of triggers grouped into a single + StreamDatum index """ @abstractmethod @@ -137,11 +139,13 @@ async def open(self, frames_per_event: int = 1) -> dict[str, DataKey]: """Open writer and wait for it to be ready for data. Args: - frames_per_event: The number of triggers are grouped into a single StreamDatum index. - A frames_per_event > 1 can be useful to have frames from a faster detector able to be zipped with a single frame from a slow detector. - E.g. if number_of_triggers=10 and frames_per_event=5 then the detector will take 10 frames, - but publish 2 StreamDatum indices, and describe() will show a shape of (5, h, w) for each. - + frames_per_event: The number of triggers that are grouped into a single + StreamDatum index. A frames_per_event > 1 can be useful to have + frames from a faster detector able to be zipped with a single frame + from a slower detector. E.g. if number_of_triggers=10 and + frames_per_event=5 then the detector will take 10 frames, but publish + 2 StreamDatum indices, and describe() will show a shape of (5, h, w) + for each. Returns: Output for ``describe()`` """ diff --git a/src/ophyd_async/epics/adaravis/_aravis_controller.py b/src/ophyd_async/epics/adaravis/_aravis_controller.py index 7fbc21c88f..e78fdf21aa 100644 --- a/src/ophyd_async/epics/adaravis/_aravis_controller.py +++ b/src/ophyd_async/epics/adaravis/_aravis_controller.py @@ -14,6 +14,7 @@ # runtime. See https://github.com/bluesky/ophyd-async/issues/308 _HIGHEST_POSSIBLE_DEADTIME = 1961e-6 + class AravisController(adcore.ADBaseController[AravisDriverIO]): GPIO_NUMBER = Literal[1, 2, 3, 4] diff --git a/src/ophyd_async/epics/adcore/_core_writer.py b/src/ophyd_async/epics/adcore/_core_writer.py index ff1a7938ab..3f50e5a242 100644 --- a/src/ophyd_async/epics/adcore/_core_writer.py +++ b/src/ophyd_async/epics/adcore/_core_writer.py @@ -185,9 +185,12 @@ async def collect_stream_docs( data_key=self._name_provider(), # Q: What are the parameters used for? Extra info? parameters={ - # TODO: Validate this assumption and that it should not be self._frames_per_event - # Assume that we always write self._frames_per_event frames per file/chunk + # Assume that we always write self._frames_per_event + # frames per file/chunk + # TODO: Validate this assumption and that it should + # not be self._frames_per_event "chunk_shape": (self._frames_per_event, *frame_shape), + "shape": (self._frames_per_event, *frame_shape), # Include file template for reconstruction in consolidator "template": file_template, }, diff --git a/src/ophyd_async/epics/adcore/_hdf_writer.py b/src/ophyd_async/epics/adcore/_hdf_writer.py index 56b3dd4c0a..ae4e11cad7 100644 --- a/src/ophyd_async/epics/adcore/_hdf_writer.py +++ b/src/ophyd_async/epics/adcore/_hdf_writer.py @@ -74,7 +74,7 @@ async def open(self, frames_per_event: int = 1) -> dict[str, DataKey]: name = self._name_provider() detector_shape = await self._dataset_describer.shape() np_dtype = await self._dataset_describer.np_datatype() - + # Used by the base class self._frames_per_event = frames_per_event @@ -86,7 +86,9 @@ async def open(self, frames_per_event: int = 1) -> dict[str, DataKey]: HDFDataset( data_key=name, dataset="/entry/data/data", - shape=(frames_per_event, *detector_shape) if frames_per_event > 1 or detector_shape else (), + shape=(frames_per_event, *detector_shape) + if frames_per_event > 1 or detector_shape + else (), dtype_numpy=np_dtype, chunk_shape=(frames_per_chunk, *detector_shape), ) diff --git a/src/ophyd_async/epics/eiger/_odin_io.py b/src/ophyd_async/epics/eiger/_odin_io.py index 1922b7f528..0cab1b25a4 100644 --- a/src/ophyd_async/epics/eiger/_odin_io.py +++ b/src/ophyd_async/epics/eiger/_odin_io.py @@ -2,7 +2,7 @@ from collections.abc import AsyncGenerator, AsyncIterator from bluesky.protocols import StreamAsset -from event_model import DataKey # type: ignore +from event_model import DataKey # type: ignore from ophyd_async.core import ( DEFAULT_TIMEOUT, @@ -102,7 +102,7 @@ async def _describe(self) -> dict[str, DataKey]: return { "data": DataKey( source=self._drv.file_name.source, - shape=list((self._frames_per_event, *data_shape)), + shape=[self._frames_per_event, *data_shape], dtype="array", # TODO: Use correct type based on eiger https://github.com/bluesky/ophyd-async/issues/529 dtype_numpy=" None: self._datasets = [ # TODO: Update chunk size to read signal once available in IOC # Currently PandA IOC sets chunk size to 1024 points per chunk - HDFDataset( - dataset_name, "/" + dataset_name, shape=(), chunk_shape=(1024,) - ) + HDFDataset(dataset_name, "/" + dataset_name, shape=(), chunk_shape=(1024,)) for dataset_name in capture_table.name ] diff --git a/src/ophyd_async/sim/_pattern_detector/_pattern_detector_writer.py b/src/ophyd_async/sim/_pattern_detector/_pattern_detector_writer.py index 2513b39559..4259194531 100644 --- a/src/ophyd_async/sim/_pattern_detector/_pattern_detector_writer.py +++ b/src/ophyd_async/sim/_pattern_detector/_pattern_detector_writer.py @@ -1,6 +1,6 @@ from collections.abc import AsyncGenerator, AsyncIterator -from event_model import DataKey # type: ignore +from event_model import DataKey # type: ignore from ophyd_async.core import DEFAULT_TIMEOUT, DetectorWriter, NameProvider, PathProvider diff --git a/tests/core/test_flyer.py b/tests/core/test_flyer.py index 29c411ce7c..4a8e7da11b 100644 --- a/tests/core/test_flyer.py +++ b/tests/core/test_flyer.py @@ -9,7 +9,11 @@ import pytest from bluesky.protocols import StreamAsset from bluesky.run_engine import RunEngine -from event_model import ComposeStreamResourceBundle, DataKey, compose_stream_resource # type: ignore +from event_model import ( # type: ignore + ComposeStreamResourceBundle, + DataKey, + compose_stream_resource, +) from pydantic import ValidationError from ophyd_async.core import ( diff --git a/tests/epics/adcore/test_writers.py b/tests/epics/adcore/test_writers.py index 01ecc963ee..8792808ae7 100644 --- a/tests/epics/adcore/test_writers.py +++ b/tests/epics/adcore/test_writers.py @@ -14,7 +14,7 @@ from ophyd_async.epics.adpilatus import PilatusReadoutTime from ophyd_async.epics.core import epics_signal_r from ophyd_async.plan_stubs import setup_ndattributes, setup_ndstats_sum -from ophyd_async.testing import set_mock_value, callback_on_mock_put +from ophyd_async.testing import callback_on_mock_put, set_mock_value class DummyDatasetDescriber(DatasetDescriber): diff --git a/tests/plan_stubs/test_fly.py b/tests/plan_stubs/test_fly.py index 1ec4843fb7..997fc8099b 100644 --- a/tests/plan_stubs/test_fly.py +++ b/tests/plan_stubs/test_fly.py @@ -6,7 +6,11 @@ import pytest from bluesky.protocols import StreamAsset from bluesky.run_engine import RunEngine -from event_model import ComposeStreamResourceBundle, DataKey, compose_stream_resource # type: ignore +from event_model import ( # type: ignore + ComposeStreamResourceBundle, + DataKey, + compose_stream_resource, +) from ophyd_async.core import ( DEFAULT_TIMEOUT, @@ -52,7 +56,7 @@ async def open(self, frames_per_event: int = 1) -> dict[str, DataKey]: return { self._name: DataKey( source="soft://some-source", - shape=list((frames_per_event, *self._shape)), + shape=[frames_per_event, *self._shape], dtype="number", external="STREAM:", ) @@ -81,6 +85,7 @@ async def collect_stream_docs( parameters={ "path": "", "swmr": False, + "shape": (self._frames_per_event, *self._shape), }, uid=None, validate=True, From 93f2f97692bb47e52353c3f74842d9f4c58a98af Mon Sep 17 00:00:00 2001 From: thomashopkins32 Date: Wed, 8 Jan 2025 14:10:44 -0500 Subject: [PATCH 50/60] Added unit tests for describe with > 1 frames_per_event --- tests/conftest.py | 6 +++++- tests/epics/adaravis/test_aravis.py | 6 ++++-- tests/epics/adkinetix/test_kinetix.py | 6 ++++-- tests/epics/advimba/test_vimba.py | 3 ++- 4 files changed, 15 insertions(+), 6 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 69d4cfdf4f..55d1910d4a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -239,13 +239,17 @@ def static_path_provider( @pytest.fixture -def one_shot_trigger_info() -> TriggerInfo: +def one_shot_trigger_info(request: FixtureRequest) -> TriggerInfo: + # If the fixture is called with a parameter, use it as the frames_per_event + # otherwise use 1 + param = getattr(request, "param", 1) return TriggerInfo( frame_timeout=None, number_of_triggers=1, trigger=DetectorTrigger.INTERNAL, deadtime=None, livetime=None, + frames_per_event=param if isinstance(param, int) else 1, ) diff --git a/tests/epics/adaravis/test_aravis.py b/tests/epics/adaravis/test_aravis.py index 63ad1f5e32..7a12932fcb 100644 --- a/tests/epics/adaravis/test_aravis.py +++ b/tests/epics/adaravis/test_aravis.py @@ -67,6 +67,7 @@ async def test_can_read(test_adaravis: adaravis.AravisDetector): assert (await test_adaravis.read()) == {} +@pytest.mark.parametrize("one_shot_trigger_info", [1, 2, 10, 100], indirect=True) async def test_decribe_describes_writer_dataset( test_adaravis: adaravis.AravisDetector, one_shot_trigger_info: TriggerInfo ): @@ -76,7 +77,7 @@ async def test_decribe_describes_writer_dataset( assert await test_adaravis.describe() == { "test_adaravis1": { "source": "mock+ca://ARAVIS1:HDF1:FullFileName_RBV", - "shape": [1, 10, 10], + "shape": [one_shot_trigger_info.frames_per_event, 10, 10], "dtype": "array", "dtype_numpy": "|i1", "external": "STREAM:", @@ -115,6 +116,7 @@ async def test_can_collect( assert stream_datum["indices"] == {"start": 0, "stop": 1} +@pytest.mark.parametrize("one_shot_trigger_info", [1, 2, 10, 100], indirect=True) async def test_can_decribe_collect( test_adaravis: adaravis.AravisDetector, one_shot_trigger_info: TriggerInfo ): @@ -124,7 +126,7 @@ async def test_can_decribe_collect( assert (await test_adaravis.describe_collect()) == { "test_adaravis1": { "source": "mock+ca://ARAVIS1:HDF1:FullFileName_RBV", - "shape": [1, 10, 10], + "shape": [one_shot_trigger_info.frames_per_event, 10, 10], "dtype": "array", "dtype_numpy": "|i1", "external": "STREAM:", diff --git a/tests/epics/adkinetix/test_kinetix.py b/tests/epics/adkinetix/test_kinetix.py index 71ab8cf26c..a1afd59daf 100644 --- a/tests/epics/adkinetix/test_kinetix.py +++ b/tests/epics/adkinetix/test_kinetix.py @@ -61,6 +61,7 @@ async def test_can_read(test_adkinetix: adkinetix.KinetixDetector): assert (await test_adkinetix.read()) == {} +@pytest.mark.parametrize("one_shot_trigger_info", [1, 2, 10, 100], indirect=True) async def test_decribe_describes_writer_dataset( test_adkinetix: adkinetix.KinetixDetector, one_shot_trigger_info: TriggerInfo ): @@ -70,7 +71,7 @@ async def test_decribe_describes_writer_dataset( assert await test_adkinetix.describe() == { "test_adkinetix1": { "source": "mock+ca://KINETIX1:HDF1:FullFileName_RBV", - "shape": [1, 10, 10], + "shape": [one_shot_trigger_info.frames_per_event, 10, 10], "dtype": "array", "dtype_numpy": "|i1", "external": "STREAM:", @@ -110,6 +111,7 @@ async def test_can_collect( assert stream_datum["indices"] == {"start": 0, "stop": 1} +@pytest.mark.parametrize("one_shot_trigger_info", [1, 2, 10, 100], indirect=True) async def test_can_decribe_collect( test_adkinetix: adkinetix.KinetixDetector, one_shot_trigger_info: TriggerInfo ): @@ -119,7 +121,7 @@ async def test_can_decribe_collect( assert (await test_adkinetix.describe_collect()) == { "test_adkinetix1": { "source": "mock+ca://KINETIX1:HDF1:FullFileName_RBV", - "shape": [1, 10, 10], + "shape": [one_shot_trigger_info.frames_per_event, 10, 10], "dtype": "array", "dtype_numpy": "|i1", "external": "STREAM:", diff --git a/tests/epics/advimba/test_vimba.py b/tests/epics/advimba/test_vimba.py index a4d808d67f..e2b08d1d14 100644 --- a/tests/epics/advimba/test_vimba.py +++ b/tests/epics/advimba/test_vimba.py @@ -128,6 +128,7 @@ async def test_can_collect( assert stream_datum["indices"] == {"start": 0, "stop": 1} +@pytest.mark.parametrize("one_shot_trigger_info", [1, 2, 10, 100], indirect=True) async def test_can_decribe_collect( test_advimba: advimba.VimbaDetector, one_shot_trigger_info: TriggerInfo ): @@ -137,7 +138,7 @@ async def test_can_decribe_collect( assert (await test_advimba.describe_collect()) == { "test_advimba1": { "source": "mock+ca://VIMBA1:HDF1:FullFileName_RBV", - "shape": [1, 10, 10], + "shape": [one_shot_trigger_info.frames_per_event, 10, 10], "dtype": "array", "dtype_numpy": "|i1", "external": "STREAM:", From 9410c4fe7efdd5acb1c1a47955cf2ccd24fbfbfc Mon Sep 17 00:00:00 2001 From: thomashopkins32 Date: Wed, 8 Jan 2025 14:19:11 -0500 Subject: [PATCH 51/60] Add unit tests for collect with > 1 frames_per_event --- tests/epics/adaravis/test_aravis.py | 3 ++- tests/epics/adkinetix/test_kinetix.py | 3 ++- tests/epics/advimba/test_vimba.py | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/epics/adaravis/test_aravis.py b/tests/epics/adaravis/test_aravis.py index 7a12932fcb..814781efca 100644 --- a/tests/epics/adaravis/test_aravis.py +++ b/tests/epics/adaravis/test_aravis.py @@ -85,6 +85,7 @@ async def test_decribe_describes_writer_dataset( } +@pytest.mark.parametrize("one_shot_trigger_info", [1, 2, 10, 100], indirect=True) async def test_can_collect( test_adaravis: adaravis.AravisDetector, static_path_provider: PathProvider, @@ -106,7 +107,7 @@ async def test_can_collect( assert stream_resource["parameters"] == { "dataset": "/entry/data/data", "swmr": False, - "shape": (1, 10, 10), + "shape": (one_shot_trigger_info.frames_per_event, 10, 10), "chunk_shape": (1, 10, 10), } assert docs[1][0] == "stream_datum" diff --git a/tests/epics/adkinetix/test_kinetix.py b/tests/epics/adkinetix/test_kinetix.py index a1afd59daf..889f8258ea 100644 --- a/tests/epics/adkinetix/test_kinetix.py +++ b/tests/epics/adkinetix/test_kinetix.py @@ -79,6 +79,7 @@ async def test_decribe_describes_writer_dataset( } +@pytest.mark.parametrize("one_shot_trigger_info", [1, 2, 10, 100], indirect=True) async def test_can_collect( test_adkinetix: adkinetix.KinetixDetector, static_path_provider: StaticPathProvider, @@ -101,7 +102,7 @@ async def test_can_collect( assert stream_resource["parameters"] == { "dataset": "/entry/data/data", "swmr": False, - "shape": (1, 10, 10), + "shape": (one_shot_trigger_info.frames_per_event, 10, 10), "chunk_shape": (1, 10, 10), } assert docs[1][0] == "stream_datum" diff --git a/tests/epics/advimba/test_vimba.py b/tests/epics/advimba/test_vimba.py index e2b08d1d14..45dbc456d7 100644 --- a/tests/epics/advimba/test_vimba.py +++ b/tests/epics/advimba/test_vimba.py @@ -96,6 +96,7 @@ async def test_decribe_describes_writer_dataset( } +@pytest.mark.parametrize("one_shot_trigger_info", [1, 2, 10, 100], indirect=True) async def test_can_collect( test_advimba: advimba.VimbaDetector, static_path_provider: PathProvider, @@ -118,7 +119,7 @@ async def test_can_collect( assert stream_resource["parameters"] == { "dataset": "/entry/data/data", "swmr": False, - "shape": (1, 10, 10), + "shape": (one_shot_trigger_info.frames_per_event, 10, 10), "chunk_shape": (1, 10, 10), } assert docs[1][0] == "stream_datum" From 9ca8f2a736bc7d0a7a9420dff5f0b09a895ed4d0 Mon Sep 17 00:00:00 2001 From: thomashopkins32 Date: Wed, 8 Jan 2025 16:13:01 -0500 Subject: [PATCH 52/60] Fix docs indentation --- src/ophyd_async/core/_detector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ophyd_async/core/_detector.py b/src/ophyd_async/core/_detector.py index b413cf7689..4695856626 100644 --- a/src/ophyd_async/core/_detector.py +++ b/src/ophyd_async/core/_detector.py @@ -110,7 +110,7 @@ async def prepare(self, trigger_info: TriggerInfo) -> None: deadtime Defaults to None. This is the minimum deadtime between triggers. frames_per_event The number of triggers grouped into a single - StreamDatum index + StreamDatum index """ @abstractmethod From 33b6b2175b4d5bbdd6e71ee205a68b181dd30a2e Mon Sep 17 00:00:00 2001 From: thomashopkins32 Date: Tue, 14 Jan 2025 11:07:58 -0500 Subject: [PATCH 53/60] Remove shape from stream resource parameters --- src/ophyd_async/core/_detector.py | 2 +- src/ophyd_async/core/_hdf_dataset.py | 1 - src/ophyd_async/epics/adcore/_core_writer.py | 12 +++--------- tests/core/test_flyer.py | 1 - tests/epics/adaravis/test_aravis.py | 1 - tests/epics/adkinetix/test_kinetix.py | 1 - tests/epics/advimba/test_vimba.py | 1 - tests/fastcs/panda/test_hdf_panda.py | 1 - tests/fastcs/panda/test_writer.py | 1 - tests/plan_stubs/test_fly.py | 1 - 10 files changed, 4 insertions(+), 18 deletions(-) diff --git a/src/ophyd_async/core/_detector.py b/src/ophyd_async/core/_detector.py index 887e4d58a1..ee87c75ee6 100644 --- a/src/ophyd_async/core/_detector.py +++ b/src/ophyd_async/core/_detector.py @@ -21,7 +21,7 @@ Triggerable, WritesStreamAssets, ) -from event_model import DataKey # type: ignore +from event_model import DataKey # type: ignore from pydantic import BaseModel, Field, NonNegativeInt, computed_field from ._device import Device, DeviceConnector diff --git a/src/ophyd_async/core/_hdf_dataset.py b/src/ophyd_async/core/_hdf_dataset.py index 6e13c58e18..a595dba471 100644 --- a/src/ophyd_async/core/_hdf_dataset.py +++ b/src/ophyd_async/core/_hdf_dataset.py @@ -66,7 +66,6 @@ def __init__( parameters={ "dataset": ds.dataset, "swmr": ds.swmr, - "shape": ds.shape, "chunk_shape": ds.chunk_shape, }, uid=None, diff --git a/src/ophyd_async/epics/adcore/_core_writer.py b/src/ophyd_async/epics/adcore/_core_writer.py index 4455e45f75..980e27143a 100644 --- a/src/ophyd_async/epics/adcore/_core_writer.py +++ b/src/ophyd_async/epics/adcore/_core_writer.py @@ -180,17 +180,11 @@ async def collect_stream_docs( self._emitted_resource = bundler_composer( mimetype=self._mimetype, uri=uri, - # TODO: This is confusing, I expected this to be of type `DataKey` - # but it is a string. Naming could be improved maybe? data_key=self._name_provider(), - # Q: What are the parameters used for? Extra info? parameters={ - # Assume that we always write self._frames_per_event - # frames per file/chunk - # TODO: Validate this assumption and that it should - # not be self._frames_per_event - "chunk_shape": (self._frames_per_event, *frame_shape), - "shape": (self._frames_per_event, *frame_shape), + # Assume that we always write 1 frame per file/chunk, this + # may change to self._frames_per_event in the future + "chunk_shape": (1, *frame_shape), # Include file template for reconstruction in consolidator "template": file_template, }, diff --git a/tests/core/test_flyer.py b/tests/core/test_flyer.py index 4a8e7da11b..2c1bf300f3 100644 --- a/tests/core/test_flyer.py +++ b/tests/core/test_flyer.py @@ -100,7 +100,6 @@ async def collect_stream_docs( parameters={ "path": "", "dataset": "", - "frames_per_event": self._frames_per_event, }, uid=None, validate=True, diff --git a/tests/epics/adaravis/test_aravis.py b/tests/epics/adaravis/test_aravis.py index 814781efca..7c9d16da50 100644 --- a/tests/epics/adaravis/test_aravis.py +++ b/tests/epics/adaravis/test_aravis.py @@ -107,7 +107,6 @@ async def test_can_collect( assert stream_resource["parameters"] == { "dataset": "/entry/data/data", "swmr": False, - "shape": (one_shot_trigger_info.frames_per_event, 10, 10), "chunk_shape": (1, 10, 10), } assert docs[1][0] == "stream_datum" diff --git a/tests/epics/adkinetix/test_kinetix.py b/tests/epics/adkinetix/test_kinetix.py index 889f8258ea..3a2f23d762 100644 --- a/tests/epics/adkinetix/test_kinetix.py +++ b/tests/epics/adkinetix/test_kinetix.py @@ -102,7 +102,6 @@ async def test_can_collect( assert stream_resource["parameters"] == { "dataset": "/entry/data/data", "swmr": False, - "shape": (one_shot_trigger_info.frames_per_event, 10, 10), "chunk_shape": (1, 10, 10), } assert docs[1][0] == "stream_datum" diff --git a/tests/epics/advimba/test_vimba.py b/tests/epics/advimba/test_vimba.py index 45dbc456d7..4c7a46b198 100644 --- a/tests/epics/advimba/test_vimba.py +++ b/tests/epics/advimba/test_vimba.py @@ -119,7 +119,6 @@ async def test_can_collect( assert stream_resource["parameters"] == { "dataset": "/entry/data/data", "swmr": False, - "shape": (one_shot_trigger_info.frames_per_event, 10, 10), "chunk_shape": (1, 10, 10), } assert docs[1][0] == "stream_datum" diff --git a/tests/fastcs/panda/test_hdf_panda.py b/tests/fastcs/panda/test_hdf_panda.py index 0cd781a51d..04b88efea8 100644 --- a/tests/fastcs/panda/test_hdf_panda.py +++ b/tests/fastcs/panda/test_hdf_panda.py @@ -146,7 +146,6 @@ def flying_plan(): "parameters": { "dataset": f"/{dataset_name}", "swmr": False, - "shape": (), "chunk_shape": (1024,), }, } diff --git a/tests/fastcs/panda/test_writer.py b/tests/fastcs/panda/test_writer.py index 94449b487f..5b18ec6cfc 100644 --- a/tests/fastcs/panda/test_writer.py +++ b/tests/fastcs/panda/test_writer.py @@ -189,7 +189,6 @@ def assert_resource_document(name, resource_doc): "parameters": { "dataset": f"/{name}", "swmr": False, - "shape": (), "chunk_shape": (1024,), }, } diff --git a/tests/plan_stubs/test_fly.py b/tests/plan_stubs/test_fly.py index 997fc8099b..ce181a3296 100644 --- a/tests/plan_stubs/test_fly.py +++ b/tests/plan_stubs/test_fly.py @@ -85,7 +85,6 @@ async def collect_stream_docs( parameters={ "path": "", "swmr": False, - "shape": (self._frames_per_event, *self._shape), }, uid=None, validate=True, From 3fde1d2e29deb2d6e139eae82d5ba39e610ba8c3 Mon Sep 17 00:00:00 2001 From: thomashopkins32 Date: Tue, 14 Jan 2025 11:09:49 -0500 Subject: [PATCH 54/60] Ruff check fixes --- src/ophyd_async/core/_detector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ophyd_async/core/_detector.py b/src/ophyd_async/core/_detector.py index ee87c75ee6..887e4d58a1 100644 --- a/src/ophyd_async/core/_detector.py +++ b/src/ophyd_async/core/_detector.py @@ -21,7 +21,7 @@ Triggerable, WritesStreamAssets, ) -from event_model import DataKey # type: ignore +from event_model import DataKey # type: ignore from pydantic import BaseModel, Field, NonNegativeInt, computed_field from ._device import Device, DeviceConnector From 161f0223997d9ae4ab63f1f080786f2717e581c7 Mon Sep 17 00:00:00 2001 From: thomashopkins32 Date: Tue, 14 Jan 2025 14:25:31 -0500 Subject: [PATCH 55/60] Make the first dimension for scalar values always the frames_per_event --- src/ophyd_async/epics/adcore/_hdf_writer.py | 8 +++----- src/ophyd_async/fastcs/panda/_writer.py | 6 +++--- .../sim/_pattern_detector/_pattern_generator.py | 4 ++-- tests/epics/adcore/test_writers.py | 4 ++-- tests/fastcs/panda/test_writer.py | 2 +- 5 files changed, 11 insertions(+), 13 deletions(-) diff --git a/src/ophyd_async/epics/adcore/_hdf_writer.py b/src/ophyd_async/epics/adcore/_hdf_writer.py index ae4e11cad7..07350d2868 100644 --- a/src/ophyd_async/epics/adcore/_hdf_writer.py +++ b/src/ophyd_async/epics/adcore/_hdf_writer.py @@ -86,9 +86,7 @@ async def open(self, frames_per_event: int = 1) -> dict[str, DataKey]: HDFDataset( data_key=name, dataset="/entry/data/data", - shape=(frames_per_event, *detector_shape) - if frames_per_event > 1 or detector_shape - else (), + shape=(frames_per_event, *detector_shape), dtype_numpy=np_dtype, chunk_shape=(frames_per_chunk, *detector_shape), ) @@ -114,7 +112,7 @@ async def open(self, frames_per_event: int = 1) -> dict[str, DataKey]: HDFDataset( datakey, f"/entry/instrument/NDAttributes/{datakey}", - (frames_per_event,) if frames_per_event > 1 else (), + (frames_per_event,), np_datatype, # NDAttributes appear to always be configured with # this chunk size @@ -126,7 +124,7 @@ async def open(self, frames_per_event: int = 1) -> dict[str, DataKey]: ds.data_key: DataKey( source=self.fileio.full_file_name.source, shape=list(ds.shape), - dtype="array" if ds.shape else "number", + dtype="array" if len(ds.shape) > 1 else "number", dtype_numpy=ds.dtype_numpy, external="STREAM:", ) diff --git a/src/ophyd_async/fastcs/panda/_writer.py b/src/ophyd_async/fastcs/panda/_writer.py index c91d64d480..2f31e369ad 100644 --- a/src/ophyd_async/fastcs/panda/_writer.py +++ b/src/ophyd_async/fastcs/panda/_writer.py @@ -84,8 +84,8 @@ async def _describe(self) -> dict[str, DataKey]: ds.data_key: DataKey( source=self.panda_data_block.hdf_directory.source, # frames_per_event is always 1 for PandA - shape=[1, *ds.shape] if ds.shape else [], - dtype="array" if ds.shape else "number", + shape=list(ds.shape), + dtype="array" if len(ds.shape) > 1 else "number", # PandA data should always be written as Float64 dtype_numpy=" None: self._datasets = [ # TODO: Update chunk size to read signal once available in IOC # Currently PandA IOC sets chunk size to 1024 points per chunk - HDFDataset(dataset_name, "/" + dataset_name, shape=(), chunk_shape=(1024,)) + HDFDataset(dataset_name, "/" + dataset_name, shape=(1,), chunk_shape=(1024,)) for dataset_name in capture_table.name ] diff --git a/src/ophyd_async/sim/_pattern_detector/_pattern_generator.py b/src/ophyd_async/sim/_pattern_detector/_pattern_generator.py index 31620b8934..7b3abd95c8 100644 --- a/src/ophyd_async/sim/_pattern_detector/_pattern_generator.py +++ b/src/ophyd_async/sim/_pattern_detector/_pattern_generator.py @@ -144,7 +144,7 @@ async def open_file( HDFDataset( f"{name}-sum", dataset=SUM_PATH, - shape=(frames_per_event,) if frames_per_event > 1 else (), + shape=(frames_per_event,), ), ] @@ -152,7 +152,7 @@ async def open_file( ds.data_key: DataKey( source="sim://pattern-generator-hdf-file", shape=list(ds.shape), - dtype="array" if ds.shape else "number", + dtype="array" if len(ds.shape) > 1 else "number", external="STREAM:", ) for ds in self._datasets diff --git a/tests/epics/adcore/test_writers.py b/tests/epics/adcore/test_writers.py index 8792808ae7..1aadd5da63 100644 --- a/tests/epics/adcore/test_writers.py +++ b/tests/epics/adcore/test_writers.py @@ -177,7 +177,7 @@ async def test_stats_describe_when_plugin_configured( }, "mydetector-sum": { "source": "mock+ca://HDF:FullFileName_RBV", - "shape": [], + "shape": [1,], "dtype": "number", "dtype_numpy": " Date: Tue, 14 Jan 2025 14:26:00 -0500 Subject: [PATCH 56/60] Ruff format --- src/ophyd_async/fastcs/panda/_writer.py | 4 +++- tests/epics/adcore/test_writers.py | 8 ++++++-- tests/fastcs/panda/test_writer.py | 4 +++- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/ophyd_async/fastcs/panda/_writer.py b/src/ophyd_async/fastcs/panda/_writer.py index 2f31e369ad..a91f9058e7 100644 --- a/src/ophyd_async/fastcs/panda/_writer.py +++ b/src/ophyd_async/fastcs/panda/_writer.py @@ -104,7 +104,9 @@ async def _update_datasets(self) -> None: self._datasets = [ # TODO: Update chunk size to read signal once available in IOC # Currently PandA IOC sets chunk size to 1024 points per chunk - HDFDataset(dataset_name, "/" + dataset_name, shape=(1,), chunk_shape=(1024,)) + HDFDataset( + dataset_name, "/" + dataset_name, shape=(1,), chunk_shape=(1024,) + ) for dataset_name in capture_table.name ] diff --git a/tests/epics/adcore/test_writers.py b/tests/epics/adcore/test_writers.py index 1aadd5da63..dad5204d6d 100644 --- a/tests/epics/adcore/test_writers.py +++ b/tests/epics/adcore/test_writers.py @@ -177,7 +177,9 @@ async def test_stats_describe_when_plugin_configured( }, "mydetector-sum": { "source": "mock+ca://HDF:FullFileName_RBV", - "shape": [1,], + "shape": [ + 1, + ], "dtype": "number", "dtype_numpy": " Date: Tue, 14 Jan 2025 14:31:48 -0500 Subject: [PATCH 57/60] Forgot one test --- tests/sim/test_sim_writer.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/sim/test_sim_writer.py b/tests/sim/test_sim_writer.py index cb65207988..ce701170f4 100644 --- a/tests/sim/test_sim_writer.py +++ b/tests/sim/test_sim_writer.py @@ -27,7 +27,9 @@ async def test_correct_descriptor_doc_after_open(writer: PatternDetectorWriter): }, "NAME-sum": { "source": "sim://pattern-generator-hdf-file", - "shape": [], + "shape": [ + 1, + ], "dtype": "number", "external": "STREAM:", }, From a5b1f27c546f0847ec27ce4e8f6cfec7e79f8e73 Mon Sep 17 00:00:00 2001 From: thomashopkins32 Date: Thu, 30 Jan 2025 10:06:28 -0500 Subject: [PATCH 58/60] Total number of triggers scaled by frames_per_event; PandA now supports frames_per_event > 1 --- src/ophyd_async/core/_detector.py | 10 +++++----- src/ophyd_async/fastcs/panda/_writer.py | 22 +++++++++++++--------- tests/fastcs/panda/test_writer.py | 9 +++++---- 3 files changed, 23 insertions(+), 18 deletions(-) diff --git a/src/ophyd_async/core/_detector.py b/src/ophyd_async/core/_detector.py index 887e4d58a1..d6c8d19e92 100644 --- a/src/ophyd_async/core/_detector.py +++ b/src/ophyd_async/core/_detector.py @@ -21,8 +21,8 @@ Triggerable, WritesStreamAssets, ) -from event_model import DataKey # type: ignore -from pydantic import BaseModel, Field, NonNegativeInt, computed_field +from event_model import DataKey +from pydantic import BaseModel, Field, NonNegativeInt, PositiveInt, computed_field from ._device import Device, DeviceConnector from ._protocol import AsyncConfigurable, AsyncReadable @@ -71,15 +71,15 @@ class TriggerInfo(BaseModel): #: number_of_triggers=10 and frames_per_event=5 then the detector will take #: 10 frames, but publish 2 StreamDatum indices, and describe() will show a #: shape of (5, h, w) for each. - frames_per_event: NonNegativeInt = 1 + frames_per_event: PositiveInt = 1 @computed_field @cached_property def total_number_of_triggers(self) -> int: return ( - sum(self.number_of_triggers) + sum(self.number_of_triggers) * self.frames_per_event if isinstance(self.number_of_triggers, list) - else self.number_of_triggers + else self.number_of_triggers * self.frames_per_event ) diff --git a/src/ophyd_async/fastcs/panda/_writer.py b/src/ophyd_async/fastcs/panda/_writer.py index a91f9058e7..d2cfcbaa72 100644 --- a/src/ophyd_async/fastcs/panda/_writer.py +++ b/src/ophyd_async/fastcs/panda/_writer.py @@ -38,6 +38,7 @@ def __init__( # Triggered on PCAP arm async def open(self, frames_per_event: int = 1) -> dict[str, DataKey]: """Retrieve and get descriptor of all PandA signals marked for capture""" + self._frames_per_event = frames_per_event # Ensure flushes are immediate await self.panda_data_block.flush_period.set(0) @@ -67,10 +68,6 @@ async def open(self, frames_per_event: int = 1) -> dict[str, DataKey]: # Wait for it to start, stashing the status that tells us when it finishes await self.panda_data_block.capture.set(True) - if frames_per_event > 1: - raise ValueError( - "All PandA datasets should be scalar, frames_per_event should be 1" - ) return await self._describe() @@ -83,7 +80,6 @@ async def _describe(self) -> dict[str, DataKey]: describe = { ds.data_key: DataKey( source=self.panda_data_block.hdf_directory.source, - # frames_per_event is always 1 for PandA shape=list(ds.shape), dtype="array" if len(ds.shape) > 1 else "number", # PandA data should always be written as Float64 @@ -105,7 +101,10 @@ async def _update_datasets(self) -> None: # TODO: Update chunk size to read signal once available in IOC # Currently PandA IOC sets chunk size to 1024 points per chunk HDFDataset( - dataset_name, "/" + dataset_name, shape=(1,), chunk_shape=(1024,) + dataset_name, + "/" + dataset_name, + shape=(self._frames_per_event,), + chunk_shape=(1024,), ) for dataset_name in capture_table.name ] @@ -124,7 +123,9 @@ async def _update_datasets(self) -> None: # StandardDetector behavior async def wait_for_index(self, index: int, timeout: float | None = DEFAULT_TIMEOUT): def matcher(value: int) -> bool: - return value >= index + # Index is already divided by frames_per_event, so we need to also + # divide the value by frames_per_event to get the correct index + return value // self._frames_per_event >= index matcher.__name__ = f"index_at_least_{index}" await wait_for_value( @@ -132,7 +133,10 @@ def matcher(value: int) -> bool: ) async def get_indices_written(self) -> int: - return await self.panda_data_block.num_captured.get_value() + return ( + await self.panda_data_block.num_captured.get_value() + // self._frames_per_event + ) async def observe_indices_written( self, timeout=DEFAULT_TIMEOUT @@ -141,7 +145,7 @@ async def observe_indices_written( async for num_captured in observe_value( self.panda_data_block.num_captured, timeout ): - yield num_captured + yield num_captured // self._frames_per_event async def collect_stream_docs( self, indices_written: int diff --git a/tests/fastcs/panda/test_writer.py b/tests/fastcs/panda/test_writer.py index 91c7a1e3e5..775cf80690 100644 --- a/tests/fastcs/panda/test_writer.py +++ b/tests/fastcs/panda/test_writer.py @@ -161,11 +161,12 @@ async def test_get_indices_written(mock_writer: PandaHDFWriter): assert written == 4 -async def test_wait_for_index(mock_writer: PandaHDFWriter): - await mock_writer.open() - set_mock_value(mock_writer.panda_data_block.num_captured, 3) +@pytest.mark.parametrize("frames_per_event", [1, 2, 11]) +async def test_wait_for_index(mock_writer: PandaHDFWriter, frames_per_event: int): + await mock_writer.open(frames_per_event=frames_per_event) + set_mock_value(mock_writer.panda_data_block.num_captured, 3 * frames_per_event) await mock_writer.wait_for_index(3, timeout=1) - set_mock_value(mock_writer.panda_data_block.num_captured, 2) + set_mock_value(mock_writer.panda_data_block.num_captured, 2 * frames_per_event) with pytest.raises(asyncio.TimeoutError): await mock_writer.wait_for_index(3, timeout=0.1) From 7c0dfa21ae23b6ebe9c48c1ed27f1d34bc32c7b4 Mon Sep 17 00:00:00 2001 From: thomashopkins32 Date: Thu, 30 Jan 2025 11:27:48 -0500 Subject: [PATCH 59/60] Fix tests --- tests/core/test_detector.py | 4 ++-- tests/epics/adandor/test_andor.py | 5 ++--- tests/epics/adcore/test_writers.py | 1 + tests/fastcs/panda/test_writer.py | 5 ----- 4 files changed, 5 insertions(+), 10 deletions(-) diff --git a/tests/core/test_detector.py b/tests/core/test_detector.py index aac3e104e8..78ca25874d 100644 --- a/tests/core/test_detector.py +++ b/tests/core/test_detector.py @@ -47,7 +47,7 @@ async def test_prepare_internal_trigger(standard_detector: StandardDetector) -> assert standard_detector._trigger_info == trigger_info assert standard_detector._number_of_triggers_iter is not None assert standard_detector._initial_frame == 0 - standard_detector._writer.open.assert_called_once_with(trigger_info.multiplier) # type: ignore + standard_detector._writer.open.assert_called_once_with(trigger_info.frames_per_event) # type: ignore standard_detector._controller.prepare.assert_called_once_with(trigger_info) # type: ignore @@ -63,7 +63,7 @@ async def test_prepare_external_trigger(standard_detector: StandardDetector) -> assert standard_detector._trigger_info == trigger_info assert standard_detector._number_of_triggers_iter is not None assert standard_detector._initial_frame == 0 - standard_detector._writer.open.assert_called_once_with(trigger_info.multiplier) # type: ignore + standard_detector._writer.open.assert_called_once_with(trigger_info.frames_per_event) # type: ignore standard_detector._controller.prepare.assert_called_once_with(trigger_info) # type: ignore standard_detector._controller.arm.assert_called_once() # type: ignore diff --git a/tests/epics/adandor/test_andor.py b/tests/epics/adandor/test_andor.py index debced3a05..50460bb887 100644 --- a/tests/epics/adandor/test_andor.py +++ b/tests/epics/adandor/test_andor.py @@ -43,7 +43,7 @@ async def test_decribe_describes_writer_dataset( assert await test_adandor.describe() == { "test_adandor21": { "source": "mock+ca://ANDOR21:HDF1:FullFileName_RBV", - "shape": [10, 10], + "shape": [1, 10, 10], "dtype": "array", "dtype_numpy": " Date: Thu, 30 Jan 2025 15:24:42 -0500 Subject: [PATCH 60/60] ruff format --- tests/core/test_detector.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/core/test_detector.py b/tests/core/test_detector.py index 78ca25874d..7dde2a34f6 100644 --- a/tests/core/test_detector.py +++ b/tests/core/test_detector.py @@ -47,7 +47,9 @@ async def test_prepare_internal_trigger(standard_detector: StandardDetector) -> assert standard_detector._trigger_info == trigger_info assert standard_detector._number_of_triggers_iter is not None assert standard_detector._initial_frame == 0 - standard_detector._writer.open.assert_called_once_with(trigger_info.frames_per_event) # type: ignore + standard_detector._writer.open.assert_called_once_with( + trigger_info.frames_per_event + ) # type: ignore standard_detector._controller.prepare.assert_called_once_with(trigger_info) # type: ignore @@ -63,7 +65,9 @@ async def test_prepare_external_trigger(standard_detector: StandardDetector) -> assert standard_detector._trigger_info == trigger_info assert standard_detector._number_of_triggers_iter is not None assert standard_detector._initial_frame == 0 - standard_detector._writer.open.assert_called_once_with(trigger_info.frames_per_event) # type: ignore + standard_detector._writer.open.assert_called_once_with( + trigger_info.frames_per_event + ) # type: ignore standard_detector._controller.prepare.assert_called_once_with(trigger_info) # type: ignore standard_detector._controller.arm.assert_called_once() # type: ignore