diff --git a/CMakeLists.txt b/CMakeLists.txt index 5d271919..313935ac 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -237,7 +237,7 @@ set(PUBLICHEADERS include/aare/Dtype.hpp include/aare/File.hpp include/aare/FileInterface.hpp - include/aare/file_utils.hpp + include/aare/RawMasterFile.hpp include/aare/Frame.hpp include/aare/json.hpp include/aare/NDArray.hpp @@ -259,7 +259,7 @@ set(SourceFiles ${CMAKE_CURRENT_SOURCE_DIR}/src/Dtype.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/Frame.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/File.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/src/file_utils.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/RawMasterFile.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/NumpyFile.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/PixelMap.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/RawFile.cpp @@ -290,7 +290,7 @@ if(AARE_TESTS) ${CMAKE_CURRENT_SOURCE_DIR}/src/defs.test.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/Dtype.test.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/Frame.test.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/src/file_utils.test.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/RawMasterFile.test.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/NDArray.test.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/NDView.test.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/ClusterFinder.test.cpp diff --git a/docs/CMakeLists.txt b/docs/CMakeLists.txt index b19a112f..f3494f1e 100644 --- a/docs/CMakeLists.txt +++ b/docs/CMakeLists.txt @@ -22,6 +22,7 @@ set(SPHINX_SOURCE_FILES src/VarClusterFinder.rst src/pyVarClusterFinder.rst src/pyFile.rst + src/pyCtbRawFile.rst ) foreach(filename ${SPHINX_SOURCE_FILES}) diff --git a/docs/src/index.rst b/docs/src/index.rst index e1ad25c6..f24170b8 100644 --- a/docs/src/index.rst +++ b/docs/src/index.rst @@ -23,4 +23,6 @@ AARE :maxdepth: 1 pyFile + pyCtbRawFile pyVarClusterFinder + diff --git a/docs/src/pyCtbRawFile.rst b/docs/src/pyCtbRawFile.rst new file mode 100644 index 00000000..6f0822d9 --- /dev/null +++ b/docs/src/pyCtbRawFile.rst @@ -0,0 +1,11 @@ + +CtbRawFile +============ + +.. py:currentmodule:: aare + +.. autoclass:: CtbRawFile + :members: + :undoc-members: + :show-inheritance: + :inherited-members: \ No newline at end of file diff --git a/include/aare/CtbRawFile.hpp b/include/aare/CtbRawFile.hpp index 8dbd1aad..24d67f52 100644 --- a/include/aare/CtbRawFile.hpp +++ b/include/aare/CtbRawFile.hpp @@ -1,19 +1,37 @@ #pragma once #include "aare/FileInterface.hpp" -#include "aare/file_utils.hpp" +#include "aare/RawMasterFile.hpp" #include "aare/Frame.hpp" #include +#include namespace aare{ class CtbRawFile{ - FileNameComponents m_fnc{}; + RawMasterFile m_master; + std::ifstream m_file; + size_t m_current_frame{0}; + size_t m_current_subfile{0}; + size_t m_num_subfiles{0}; public: CtbRawFile(const std::filesystem::path &fname); + void read_into(std::byte *image_buf, DetectorHeader* header = nullptr); + void seek(size_t frame_index); //!< seek to the given frame index + size_t tell() const; //!< get the frame index of the file pointer + // in the specific class we can expose more functionality + + size_t image_size_in_bytes() const { return m_master.image_size_in_bytes(); } + +private: + void find_subfiles(); + size_t sub_file_index(size_t frame_index) const { + return frame_index / m_master.max_frames_per_file(); + } + void open_data_file(size_t subfile_index); }; diff --git a/include/aare/RawFile.hpp b/include/aare/RawFile.hpp index cd87f2a5..f9414b6c 100644 --- a/include/aare/RawFile.hpp +++ b/include/aare/RawFile.hpp @@ -180,9 +180,9 @@ class RawFile : public FileInterface { /** * @brief read the header of the file * @param fname path to the data subfile - * @return sls_detector_header + * @return DetectorHeader */ - static sls_detector_header read_header(const std::filesystem::path &fname); + static DetectorHeader read_header(const std::filesystem::path &fname); /** * @brief open the subfiles diff --git a/include/aare/RawMasterFile.hpp b/include/aare/RawMasterFile.hpp new file mode 100644 index 00000000..57d9db5a --- /dev/null +++ b/include/aare/RawMasterFile.hpp @@ -0,0 +1,96 @@ +#pragma once +#include "aare/defs.hpp" +#include +#include +#include + +#include +using json = nlohmann::json; + +namespace aare { + +class RawFileNameComponents { + std::filesystem::path m_base_path{}; + std::string m_base_name{}; + std::string m_ext{}; + int m_file_index{}; // TODO! is this measurement_index? + + public: + RawFileNameComponents(const std::filesystem::path &fname); + std::filesystem::path master_fname() const { + return m_base_path / + fmt::format("{}_master_{}{}", m_base_name, m_file_index, m_ext); + } + + std::filesystem::path data_fname(size_t mod_id, size_t file_id) { + return m_base_path / fmt::format("{}_d{}_f{}_{}.raw", m_base_name, + mod_id, file_id, m_file_index); + } + + const std::filesystem::path &base_path() const { return m_base_path; } + const std::string &base_name() const { return m_base_name; } + const std::string &ext() const { return m_ext; } + int file_index() const { return m_file_index; } +}; + +class RawMasterFile { + RawFileNameComponents m_fnc; + std::string m_version; + DetectorType m_type; + TimingMode m_timing_mode; + + size_t m_image_size_in_bytes; + size_t m_frames_in_file; + size_t m_pixels_y; + size_t m_pixels_x; + size_t m_bitdepth; + + size_t m_max_frames_per_file; + uint32_t m_adc_mask; + FrameDiscardPolicy m_frame_discard_policy; + size_t m_frame_padding; + + std::optional m_analog_samples; + std::optional m_digital_samples; + + public: + RawMasterFile(const std::filesystem::path &fpath) : m_fnc(fpath) { + if (!std::filesystem::exists(fpath)) { + throw std::runtime_error(LOCATION + " File does not exist"); + } + if (m_fnc.ext() == ".json") { + parse_json(fpath); + } else if (m_fnc.ext() == ".raw") { + parse_raw(fpath); + } else { + throw std::runtime_error(LOCATION + "Unsupported file type"); + } + } + + const std::string &version() const { return m_version; } + const DetectorType &detector_type() const { return m_type; } + const TimingMode &timing_mode() const { return m_timing_mode; } + size_t image_size_in_bytes() const { return m_image_size_in_bytes; } + size_t frames_in_file() const { return m_frames_in_file; } + size_t pixels_y() const { return m_pixels_y; } + size_t pixels_x() const { return m_pixels_x; } + size_t max_frames_per_file() const { return m_max_frames_per_file; } + size_t bitdepth() const { return m_bitdepth; } + size_t frame_padding() const { return m_frame_padding; } + const FrameDiscardPolicy &frame_discard_policy() const { + return m_frame_discard_policy; + } + + std::optional analog_samples() const { return m_analog_samples; } + std::optional digital_samples() const { return m_digital_samples; } + + std::filesystem::path data_fname(size_t mod_id, size_t file_id) { + return m_fnc.data_fname(mod_id, file_id); + } + + private: + void parse_json(const std::filesystem::path &fpath); + void parse_raw(const std::filesystem::path &fpath); +}; + +} // namespace aare \ No newline at end of file diff --git a/include/aare/SubFile.hpp b/include/aare/SubFile.hpp index 745fb808..7e59d3e1 100644 --- a/include/aare/SubFile.hpp +++ b/include/aare/SubFile.hpp @@ -15,7 +15,7 @@ namespace aare { */ class SubFile { public: - size_t write_part(std::byte *buffer, sls_detector_header header, size_t frame_index); + size_t write_part(std::byte *buffer, DetectorHeader header, size_t frame_index); /** * @brief SubFile constructor * @param fname path to the subfile diff --git a/include/aare/defs.hpp b/include/aare/defs.hpp index 434f533e..f0315fe9 100644 --- a/include/aare/defs.hpp +++ b/include/aare/defs.hpp @@ -111,7 +111,7 @@ class Cluster { /** * @brief header contained in parts of frames */ -struct sls_detector_header { +struct DetectorHeader { uint64_t frameNumber; uint32_t expLength; uint32_t packetNumber; @@ -166,6 +166,7 @@ using xy = t_xy; using dynamic_shape = std::vector; +//TODO! Can we uniform enums between the libraries? enum class DetectorType { Jungfrau, Eiger, @@ -178,6 +179,7 @@ enum class DetectorType { }; enum class TimingMode { Auto, Trigger }; +enum class FrameDiscardPolicy { NoDiscard, Discard }; template T StringTo(const std::string &arg) { return T(arg); } @@ -188,6 +190,8 @@ template <> std::string ToString(DetectorType arg); template <> TimingMode StringTo(const std::string & /*mode*/); +template <> FrameDiscardPolicy StringTo(const std::string & /*mode*/); + using DataTypeVariants = std::variant; } // namespace aare \ No newline at end of file diff --git a/include/aare/file_utils.hpp b/include/aare/file_utils.hpp deleted file mode 100644 index abccfb6b..00000000 --- a/include/aare/file_utils.hpp +++ /dev/null @@ -1,119 +0,0 @@ -#pragma once -#include "aare/defs.hpp" -#include -#include -#include - -#include -using json = nlohmann::json; - -namespace aare { -bool is_master_file(const std::filesystem::path &fpath); - - -struct FileNameComponents { - std::filesystem::path base_path{}; - std::string base_name{}; - std::string ext{}; - int findex{}; - bool valid{false}; // TODO! how do we do error handling? - - std::filesystem::path master_fname() const { - return base_path / - fmt::format("{}_master_{}{}", base_name, findex, ext); - } - - std::filesystem::path data_fname(size_t mod_id, size_t file_id) { - return base_path / fmt::format("{}_d{}_f{}_{}.raw", base_name, file_id, - mod_id, findex); - } -}; - -FileNameComponents parse_fname(const std::filesystem::path &fname); - -class MasterFile { - FileNameComponents m_fnc; - std::string m_version; - DetectorType m_type; - TimingMode m_timing_mode; - size_t m_total_frames; - size_t m_subfile_rows; - size_t m_subfile_cols; - size_t m_bitdepth; - size_t m_analog_samples; - size_t m_digital_samples; - size_t m_max_frames_per_file; - uint32_t m_adc_mask; -public: - MasterFile(const std::filesystem::path &fpath) { - m_fnc = parse_fname(fpath); - - - - std::ifstream ifs(fpath); - json j; - ifs >> j; - double v = j["Version"]; - m_version = fmt::format("{:.1f}", v); - - m_type = StringTo(j["Detector// Type"].get()); - m_timing_mode = - StringTo(j["Timing Mode"].get()); - m_total_frames = j["Frames in File"]; - m_subfile_rows = j["Pixels"]["y"]; - m_subfile_cols = j["Pixels"]["x"]; - m_max_frames_per_file = j["Max Frames Per File"]; - try { - m_bitdepth = j.at("Dynamic Range"); - } catch (const json::out_of_range &e) { - m_bitdepth = 16; - } - - try { - m_analog_samples = j.at("Analog Samples"); - }catch (const json::out_of_range &e) { - m_analog_samples = 0; - } - try{ - std::string adc_mask = j.at("ADC Mask"); - m_adc_mask = std::stoul(adc_mask, nullptr, 16); - }catch (const json::out_of_range &e) { - m_adc_mask = 0; - } - - try { - m_digital_samples = j.at("Digital Samples"); - }catch (const json::out_of_range &e) { - m_digital_samples = 0; - } - - //Update detector type for Moench - //TODO! How does this work with old .raw master files? - if (m_type == DetectorType::Moench && m_analog_samples == 0 && - m_subfile_rows == 400) { - m_type = DetectorType::Moench03; - }else if (m_type == DetectorType::Moench && m_subfile_rows == 400 && - m_analog_samples == 5000) { - m_type = DetectorType::Moench03_old; - } - - // //Here we know we have a ChipTestBoard file update the geometry? - // //TODO! Carry on information about digtial, and transceivers - // if (m_type == DetectorType::ChipTestBoard) { - // subfile_rows = 1; - // subfile_cols = m_analog_samples*__builtin_popcount(m_adc_mask); - // } - - // // only Eiger had quad - // if (m_type == DetectorType::Eiger) { - // quad = (j["Quad"] == 1); - // } - - // m_geometry = {j["Geometry"]["y"], j["Geometry"]["x"]}; - } - -}; - - - -} // namespace aare \ No newline at end of file diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index a2247f79..a8edf26e 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -27,6 +27,8 @@ target_link_libraries(_aare PRIVATE aare_core aare_compiler_flags) # List of python files to be copied to the build directory set( PYTHON_FILES aare/__init__.py + aare/CtbRawFile.py + aare/transform.py ) # Copy the python files to the build directory diff --git a/python/aare/CtbRawFile.py b/python/aare/CtbRawFile.py new file mode 100644 index 00000000..abd25661 --- /dev/null +++ b/python/aare/CtbRawFile.py @@ -0,0 +1,69 @@ + +from . import _aare +import numpy as np + +class CtbRawFile(_aare.CtbRawFile): + def __init__(self, fname, transform = None): + super().__init__(fname) + self.transform = transform + + + def read_frame(self, frame_index = None): + """Read one frame from the file. + + Args: + frame_index (int): If not None, seek to this frame before reading. + + Returns: + tuple: header, data + """ + if frame_index is not None: + self.seek(frame_index) + + header, data = super().read_frame() + if header.shape == (1,): + header = header[0] + + + if self.transform: + res = self.transform(data) + if isinstance(res, tuple): + return header, *res + else: + return header, res + + def read_n(self, n_frames): + # Do the first read to figure out what we have + tmp_header, tmp_data = self.read_frame() + + # Allocate arrays for + header = np.zeros(n_frames, dtype = tmp_header.dtype) + data = np.zeros((n_frames, *tmp_data.shape), dtype = tmp_data.dtype) + + # Copy the first frame + header[0] = tmp_header + data[0] = tmp_data + + # Do the rest of the reading + for i in range(1, n_frames): + header[i], data[i] = self.read_frame() + + return header, data + + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + pass + + def __iter__(self): + return self + + def __next__(self): + try: + return self.read_frame() + except RuntimeError: + # TODO! find a good way to check that we actually have the right exception + raise StopIteration + diff --git a/python/aare/__init__.py b/python/aare/__init__.py index 2e639125..82461af9 100644 --- a/python/aare/__init__.py +++ b/python/aare/__init__.py @@ -1,4 +1,5 @@ # Make the compiled classes that live in _aare available from aare. -from ._aare import File -from ._aare import VarClusterFinder -from ._aare import GenerateMoench03PixelMap, GenerateMoench05PixelMap \ No newline at end of file +from . import _aare + +from ._aare import VarClusterFinder, File +from .CtbRawFile import CtbRawFile \ No newline at end of file diff --git a/python/aare/transform.py b/python/aare/transform.py new file mode 100644 index 00000000..17e520ef --- /dev/null +++ b/python/aare/transform.py @@ -0,0 +1,15 @@ +import numpy as np +from . import _aare + + +class Moench05Transform: + #Could be moved to C++ without changing the interface + def __init__(self): + print('map created') + self.pixel_map = _aare.GenerateMoench05PixelMap() + + def __call__(self, data): + return np.take(data.view(np.uint16), self.pixel_map) + + +moench05 = Moench05Transform() diff --git a/python/examples/play.py b/python/examples/play.py index bff49604..857e32a2 100644 --- a/python/examples/play.py +++ b/python/examples/play.py @@ -3,6 +3,10 @@ plt.ion() import aare +from aare import CtbRawFile +print('aare imported') +from aare import transform +print('transform imported') from pathlib import Path # p = Path('/Users/erik/data/aare_test_data/jungfrau/jungfrau_single_master_0.json') @@ -15,11 +19,14 @@ # fpath = Path('/Users/erik/data/Moench03old/test_034_irradiated_noise_g4_hg_exptime_2000us_master_0.json') -fpath = Path('/Users/erik/data/Moench05/moench05_master_0.json') -f = aare.File(fpath) -f.seek(437) -frame = f.read_frame() +fpath = Path('/Users/erik/data/Moench05/moench05_multifile_master_0.json') -m = aare.GenerateMoench05PixelMap() -img = np.take(frame, m.astype(np.int64)) + +# f = aare.CtbRawFile(fpath, transform = transform.moench05) + + + +with CtbRawFile(fpath, transform = transform.moench05) as f: + for header, image in f: + print(f'Frame number: {header["frameNumber"]}') diff --git a/python/src/file.hpp b/python/src/file.hpp index 88bf7476..d4032d59 100644 --- a/python/src/file.hpp +++ b/python/src/file.hpp @@ -1,32 +1,63 @@ -#include "aare/Frame.hpp" -#include "aare/File.hpp" #include "aare/CtbRawFile.hpp" +#include "aare/File.hpp" +#include "aare/Frame.hpp" #include "aare/defs.hpp" // #include "aare/fClusterFileV2.hpp" #include #include -#include #include +#include #include #include #include #include namespace py = pybind11; -using namespace::aare; +using namespace ::aare; void define_file_io_bindings(py::module &m) { + PYBIND11_NUMPY_DTYPE(DetectorHeader, frameNumber, expLength, packetNumber, + bunchId, timestamp, modId, row, column, reserved, + debug, roundRNumber, detType, version, packetMask); + py::class_(m, "CtbRawFile") - .def(py::init()); - + .def(py::init()) + .def("read_frame", + [](CtbRawFile &self) { + size_t image_size = self.image_size_in_bytes(); + py::array image; + std::vector shape; + shape.reserve(2); + shape.push_back(1); + shape.push_back(image_size); + + py::array_t header(1); + + + // always read bytes + image = py::array_t(shape); + + self.read_into( + reinterpret_cast(image.mutable_data()), + header.mutable_data()); + + return py::make_tuple(header, image); + }) + .def("seek", &CtbRawFile::seek) + .def("tell", &CtbRawFile::tell); py::class_(m, "File") - .def(py::init([](const std::filesystem::path &fname) { return File(fname, "r", {}); })) - .def( - py::init([](const std::filesystem::path &fname, const std::string &mode) { return File(fname, mode, {}); })) - .def(py::init()) + .def(py::init([](const std::filesystem::path &fname) { + return File(fname, "r", {}); + })) + .def(py::init( + [](const std::filesystem::path &fname, const std::string &mode) { + return File(fname, mode, {}); + })) + .def(py::init()) .def("frame_number", &File::frame_number) .def_property_readonly("bytes_per_frame", &File::bytes_per_frame) @@ -38,44 +69,48 @@ void define_file_io_bindings(py::module &m) { .def_property_readonly("cols", &File::cols) .def_property_readonly("bitdepth", &File::bitdepth) .def_property_readonly("bytes_per_pixel", &File::bytes_per_pixel) - .def_property_readonly("detector_type", [](File &self){ - return ToString(self.detector_type()); - }) - .def("read_frame", [](File &self) { - const uint8_t item_size = self.bytes_per_pixel(); - py::array image; - std::vector shape; - shape.reserve(2); - shape.push_back(self.rows()); - shape.push_back(self.cols()); - if (item_size == 1) { - image = py::array_t(shape); - } else if (item_size == 2) { - image = py::array_t(shape); - } else if (item_size == 4) { - image = py::array_t(shape); - } - self.read_into(reinterpret_cast(image.mutable_data())); - return image; - }) - .def("read_frame", [](File &self, size_t frame_number) { - self.seek(frame_number); - const uint8_t item_size = self.bytes_per_pixel(); - py::array image; - std::vector shape; - shape.reserve(2); - shape.push_back(self.rows()); - shape.push_back(self.cols()); - if (item_size == 1) { - image = py::array_t(shape); - } else if (item_size == 2) { - image = py::array_t(shape); - } else if (item_size == 4) { - image = py::array_t(shape); - } - self.read_into(reinterpret_cast(image.mutable_data())); - return image; - }) + .def_property_readonly( + "detector_type", + [](File &self) { return ToString(self.detector_type()); }) + .def("read_frame", + [](File &self) { + const uint8_t item_size = self.bytes_per_pixel(); + py::array image; + std::vector shape; + shape.reserve(2); + shape.push_back(self.rows()); + shape.push_back(self.cols()); + if (item_size == 1) { + image = py::array_t(shape); + } else if (item_size == 2) { + image = py::array_t(shape); + } else if (item_size == 4) { + image = py::array_t(shape); + } + self.read_into( + reinterpret_cast(image.mutable_data())); + return image; + }) + .def("read_frame", + [](File &self, size_t frame_number) { + self.seek(frame_number); + const uint8_t item_size = self.bytes_per_pixel(); + py::array image; + std::vector shape; + shape.reserve(2); + shape.push_back(self.rows()); + shape.push_back(self.cols()); + if (item_size == 1) { + image = py::array_t(shape); + } else if (item_size == 2) { + image = py::array_t(shape); + } else if (item_size == 4) { + image = py::array_t(shape); + } + self.read_into( + reinterpret_cast(image.mutable_data())); + return image; + }) .def("read_n", [](File &self, size_t n_frames) { const uint8_t item_size = self.bytes_per_pixel(); py::array image; @@ -91,7 +126,8 @@ void define_file_io_bindings(py::module &m) { } else if (item_size == 4) { image = py::array_t(shape); } - self.read_into(reinterpret_cast(image.mutable_data()), n_frames); + self.read_into(reinterpret_cast(image.mutable_data()), + n_frames); return image; }); @@ -107,37 +143,44 @@ void define_file_io_bindings(py::module &m) { .def_readwrite("dtype", &FileConfig::dtype) .def("__eq__", &FileConfig::operator==) .def("__ne__", &FileConfig::operator!=) - .def("__repr__", [](const FileConfig &a) { return ""; }); + .def("__repr__", [](const FileConfig &a) { + return ""; + }); // py::class_(m, "ClusterHeader") // .def(py::init<>()) // .def_readwrite("frame_number", &ClusterHeader::frame_number) // .def_readwrite("n_clusters", &ClusterHeader::n_clusters) - // .def("__repr__", [](const ClusterHeader &a) { return ""; }); + // .def("__repr__", [](const ClusterHeader &a) { return ""; }); // py::class_(m, "ClusterV2_") // .def(py::init<>()) // .def_readwrite("x", &ClusterV2_::x) // .def_readwrite("y", &ClusterV2_::y) // .def_readwrite("data", &ClusterV2_::data) - // .def("__repr__", [](const ClusterV2_ &a) { return ""; }); + // .def("__repr__", [](const ClusterV2_ &a) { return ""; }); // py::class_(m, "ClusterV2") // .def(py::init<>()) // .def_readwrite("cluster", &ClusterV2::cluster) // .def_readwrite("frame_number", &ClusterV2::frame_number) - // .def("__repr__", [](const ClusterV2 &a) { return ""; }); + // .def("__repr__", [](const ClusterV2 &a) { return ""; }); // py::class_(m, "ClusterFileV2") // .def(py::init()) // .def("read", py::overload_cast<>(&ClusterFileV2::read)) // .def("read", py::overload_cast(&ClusterFileV2::read)) // .def("frame_number", &ClusterFileV2::frame_number) - // .def("write", py::overload_cast const &>(&ClusterFileV2::write)) - + // .def("write", py::overload_cast const + // &>(&ClusterFileV2::write)) + // .def("close", &ClusterFileV2::close); - // m.def("to_clustV2", [](std::vector &clusters, const int frame_number) { + // m.def("to_clustV2", [](std::vector &clusters, const int + // frame_number) { // std::vector clusters_; // for (auto &c : clusters) { // ClusterV2 cluster; diff --git a/src/CtbRawFile.cpp b/src/CtbRawFile.cpp index b70b44fd..a5ae2c79 100644 --- a/src/CtbRawFile.cpp +++ b/src/CtbRawFile.cpp @@ -1,22 +1,72 @@ #include "aare/CtbRawFile.hpp" +#include +namespace aare { +CtbRawFile::CtbRawFile(const std::filesystem::path &fname) : m_master(fname) { -namespace aare{ + + + if (m_master.detector_type() != DetectorType::ChipTestBoard) { + throw std::runtime_error(LOCATION + "Not a Ctb file"); + } + + find_subfiles(); + + // open the first subfile + m_file.open(m_master.data_fname(0, 0), std::ios::binary); +} -CtbRawFile::CtbRawFile(const std::filesystem::path &fname){ +size_t CtbRawFile::tell() const { return m_current_frame; } - if(!std::filesystem::exists(fname)){ - throw std::runtime_error(LOCATION + "File does not exist"); +void CtbRawFile::seek(size_t frame_number) { + if (auto index = sub_file_index(frame_number); index != m_current_subfile) { + open_data_file(index); + } + size_t frame_number_in_file = frame_number % m_master.max_frames_per_file(); + m_file.seekg((sizeof(DetectorHeader)+m_master.image_size_in_bytes()) * frame_number_in_file); + m_current_frame = frame_number; +} + +void CtbRawFile::find_subfiles() { + // we can semi safely assume that there is only one module for CTB + while (std::filesystem::exists(m_master.data_fname(0, m_num_subfiles))) + m_num_subfiles++; + + fmt::print("Found {} subfiles\n", m_num_subfiles); +} + +void CtbRawFile::open_data_file(size_t subfile_index) { + if (subfile_index >= m_num_subfiles) { + throw std::runtime_error(LOCATION + "Subfile index out of range"); + } + m_current_subfile = subfile_index; + m_file = std::ifstream(m_master.data_fname(0, subfile_index), std::ios::binary); // only one module for CTB + if (!m_file.is_open()) { + throw std::runtime_error(LOCATION + "Could not open data file"); + } +} + +void CtbRawFile::read_into(std::byte *image_buf, DetectorHeader* header) { + if(m_current_frame >= m_master.frames_in_file()){ + throw std::runtime_error(LOCATION + "End of file reached"); + } + + if(m_current_frame != 0 && m_current_frame % m_master.max_frames_per_file() == 0){ + open_data_file(m_current_subfile+1); } - m_fnc = parse_fname(fname); - if(!m_fnc.valid){ - throw std::runtime_error(LOCATION + "Could not parse master file name"); + if(header){ + m_file.read(reinterpret_cast(header), sizeof(DetectorHeader)); + }else{ + m_file.seekg(sizeof(DetectorHeader), std::ios::cur); } + m_file.read(reinterpret_cast(image_buf), m_master.image_size_in_bytes()); + m_current_frame++; -} + +} } // namespace aare \ No newline at end of file diff --git a/src/RawFile.cpp b/src/RawFile.cpp index ef623ce2..653e842f 100644 --- a/src/RawFile.cpp +++ b/src/RawFile.cpp @@ -45,8 +45,8 @@ void RawFile::open_subfiles() { } } -sls_detector_header RawFile::read_header(const std::filesystem::path &fname) { - sls_detector_header h{}; +DetectorHeader RawFile::read_header(const std::filesystem::path &fname) { + DetectorHeader h{}; FILE *fp = fopen(fname.string().c_str(), "r"); if (!fp) throw std::runtime_error(fmt::format("Could not open: {} for reading", fname.string())); diff --git a/src/RawMasterFile.cpp b/src/RawMasterFile.cpp new file mode 100644 index 00000000..9ad70835 --- /dev/null +++ b/src/RawMasterFile.cpp @@ -0,0 +1,103 @@ +#include "aare/RawMasterFile.hpp" + +namespace aare { + + +RawFileNameComponents::RawFileNameComponents(const std::filesystem::path &fname) { + m_base_path = fname.parent_path(); + m_base_name = fname.stem(); + m_ext = fname.extension(); + + //parse file index + try { + auto pos = m_base_name.rfind('_'); + m_file_index = std::stoi(m_base_name.substr(pos + 1)); + } catch (const std::invalid_argument &e) { + throw std::runtime_error(LOCATION + "Could not parse file index"); + } + + //remove master from base name + auto pos = m_base_name.find("_master_"); + if (pos != std::string::npos) { + m_base_name.erase(pos); + }else{ + throw std::runtime_error(LOCATION + "Could not find _master_ in file name"); + } +} + +void RawMasterFile::parse_json(const std::filesystem::path &fpath) { + std::ifstream ifs(fpath); + json j; + ifs >> j; + double v = j["Version"]; + m_version = fmt::format("{:.1f}", v); + + m_type = StringTo(j["Detector Type"].get()); + m_timing_mode = + StringTo(j["Timing Mode"].get()); + + m_image_size_in_bytes = j["Image Size in bytes"]; + m_frames_in_file = j["Frames in File"]; + m_pixels_y = j["Pixels"]["y"]; + m_pixels_x = j["Pixels"]["x"]; + + m_max_frames_per_file = j["Max Frames Per File"]; + + //Not all detectors write the bitdepth but in case + //its not there it is 16 + try { + m_bitdepth = j.at("Dynamic Range"); + } catch (const json::out_of_range &e) { + m_bitdepth = 16; + } + + m_frame_padding = j["Frame Padding"]; + m_frame_discard_policy = StringTo( + j["Frame Discard Policy"].get()); + + try { + m_analog_samples = j.at("Analog Samples"); + }catch (const json::out_of_range &e) { + // m_analog_samples = 0; + } + // try{ + // std::string adc_mask = j.at("ADC Mask"); + // m_adc_mask = std::stoul(adc_mask, nullptr, 16); + // }catch (const json::out_of_range &e) { + // m_adc_mask = 0; + // } + + try { + m_digital_samples = j.at("Digital Samples"); + }catch (const json::out_of_range &e) { + // m_digital_samples = 0; + } + + // //Update detector type for Moench + // //TODO! How does this work with old .raw master files? + // if (m_type == DetectorType::Moench && m_analog_samples == 0 && + // m_subfile_rows == 400) { + // m_type = DetectorType::Moench03; + // }else if (m_type == DetectorType::Moench && m_subfile_rows == 400 && + // m_analog_samples == 5000) { + // m_type = DetectorType::Moench03_old; + // } + + // //Here we know we have a ChipTestBoard file update the geometry? + // //TODO! Carry on information about digtial, and transceivers + // if (m_type == DetectorType::ChipTestBoard) { + // subfile_rows = 1; + // subfile_cols = m_analog_samples*__builtin_popcount(m_adc_mask); + // } + + // // only Eiger had quad + // if (m_type == DetectorType::Eiger) { + // quad = (j["Quad"] == 1); + // } + + // m_geometry = {j["Geometry"]["y"], j["Geometry"]["x"]}; +} +void RawMasterFile::parse_raw(const std::filesystem::path &fpath) { + throw std::runtime_error("Not implemented"); +} +} // namespace aare \ No newline at end of file diff --git a/src/RawMasterFile.test.cpp b/src/RawMasterFile.test.cpp new file mode 100644 index 00000000..ef84b851 --- /dev/null +++ b/src/RawMasterFile.test.cpp @@ -0,0 +1,167 @@ +#include "aare/RawMasterFile.hpp" + +#include +#include "test_config.hpp" + +using namespace aare; + + +TEST_CASE("Parse a master file fname"){ + RawFileNameComponents m("test_master_1.json"); + REQUIRE(m.base_name() == "test"); + REQUIRE(m.ext() == ".json"); + REQUIRE(m.file_index() == 1); +} + +TEST_CASE("Construction of master file name and data files"){ + RawFileNameComponents m("test_master_1.json"); + REQUIRE(m.master_fname() == "test_master_1.json"); + REQUIRE(m.data_fname(0, 0) == "test_d0_f0_1.raw"); + REQUIRE(m.data_fname(1, 0) == "test_d1_f0_1.raw"); + REQUIRE(m.data_fname(0, 1) == "test_d0_f1_1.raw"); + REQUIRE(m.data_fname(1, 1) == "test_d1_f1_1.raw"); +} + + +TEST_CASE("Parse a master file"){ + auto fpath = test_data_path() / "jungfrau" / "jungfrau_single_master_0.json"; + REQUIRE(std::filesystem::exists(fpath)); + RawMasterFile f(fpath); + + // "Version": 7.2, + REQUIRE(f.version() == "7.2"); + // "Timestamp": "Tue Feb 20 08:28:24 2024", + // "Detector Type": "Jungfrau", + REQUIRE(f.detector_type() == DetectorType::Jungfrau); + // "Timing Mode": "auto", + REQUIRE(f.timing_mode() == TimingMode::Auto); + // "Geometry": { + // "x": 1, + // "y": 1 + // }, + + // "Image Size in bytes": 1048576, + REQUIRE(f.image_size_in_bytes() == 1048576); + // "Pixels": { + // "x": 1024, + REQUIRE(f.pixels_x() == 1024); + // "y": 512 + REQUIRE(f.pixels_y() == 512); + // }, + + // "Max Frames Per File": 3, + REQUIRE(f.max_frames_per_file() == 3); + + //Jungfrau doesn't write but it is 16 + REQUIRE(f.bitdepth() == 16); + // "Frame Discard Policy": "nodiscard", + // "Frame Padding": 1, + // "Scan Parameters": "[disabled]", + // "Total Frames": 10, + // "Receiver Roi": { + // "xmin": 4294967295, + // "xmax": 4294967295, + // "ymin": 4294967295, + // "ymax": 4294967295 + // }, + // "Exptime": "10us", + // "Period": "1ms", + // "Number of UDP Interfaces": 1, + // "Number of rows": 512, + // "Frames in File": 10, + // "Frame Header Format": { + // "Frame Number": "8 bytes", + // "SubFrame Number/ExpLength": "4 bytes", + // "Packet Number": "4 bytes", + // "Bunch ID": "8 bytes", + // "Timestamp": "8 bytes", + // "Module Id": "2 bytes", + // "Row": "2 bytes", + // "Column": "2 bytes", + // "Reserved": "2 bytes", + // "Debug": "4 bytes", + // "Round Robin Number": "2 bytes", + // "Detector Type": "1 byte", + // "Header Version": "1 byte", + // "Packets Caught Mask": "64 bytes" + // } + // } + + REQUIRE_FALSE(f.analog_samples()); + REQUIRE_FALSE(f.digital_samples()); + +} + + +TEST_CASE("Read eiger master file"){ +auto fpath = test_data_path() / "eiger" / "eiger_500k_32bit_master_0.json"; + REQUIRE(std::filesystem::exists(fpath)); + RawMasterFile f(fpath); + +// { +// "Version": 7.2, +REQUIRE(f.version() == "7.2"); +// "Timestamp": "Tue Mar 26 17:24:34 2024", +// "Detector Type": "Eiger", +REQUIRE(f.detector_type() == DetectorType::Eiger); +// "Timing Mode": "auto", +REQUIRE(f.timing_mode() == TimingMode::Auto); +// "Geometry": { +// "x": 2, +// "y": 2 +// }, +// "Image Size in bytes": 524288, +REQUIRE(f.image_size_in_bytes() == 524288); +// "Pixels": { +// "x": 512, +REQUIRE(f.pixels_x() == 512); +// "y": 256 +REQUIRE(f.pixels_y() == 256); +// }, +// "Max Frames Per File": 10000, +REQUIRE(f.max_frames_per_file() == 10000); +// "Frame Discard Policy": "nodiscard", +REQUIRE(f.frame_discard_policy() == FrameDiscardPolicy::NoDiscard); +// "Frame Padding": 1, +REQUIRE(f.frame_padding() == 1); + +// "Scan Parameters": "[disabled]", +// "Total Frames": 3, +// "Receiver Roi": { +// "xmin": 4294967295, +// "xmax": 4294967295, +// "ymin": 4294967295, +// "ymax": 4294967295 +// }, +// "Dynamic Range": 32, +// "Ten Giga": 0, +// "Exptime": "5s", +// "Period": "1s", +// "Threshold Energy": -1, +// "Sub Exptime": "2.62144ms", +// "Sub Period": "2.62144ms", +// "Quad": 0, +// "Number of rows": 256, +// "Rate Corrections": "[0, 0]", +// "Frames in File": 3, +// "Frame Header Format": { +// "Frame Number": "8 bytes", +// "SubFrame Number/ExpLength": "4 bytes", +// "Packet Number": "4 bytes", +// "Bunch ID": "8 bytes", +// "Timestamp": "8 bytes", +// "Module Id": "2 bytes", +// "Row": "2 bytes", +// "Column": "2 bytes", +// "Reserved": "2 bytes", +// "Debug": "4 bytes", +// "Round Robin Number": "2 bytes", +// "Detector Type": "1 byte", +// "Header Version": "1 byte", +// "Packets Caught Mask": "64 bytes" +// } +// } + + + +} \ No newline at end of file diff --git a/src/SubFile.cpp b/src/SubFile.cpp index 6c1d0cc8..8caac96d 100644 --- a/src/SubFile.cpp +++ b/src/SubFile.cpp @@ -17,7 +17,7 @@ SubFile::SubFile(const std::filesystem::path &fname, DetectorType detector, size } if (std::filesystem::exists(fname)) { - n_frames = std::filesystem::file_size(fname) / (sizeof(sls_detector_header) + rows * cols * bitdepth / 8); + n_frames = std::filesystem::file_size(fname) / (sizeof(DetectorHeader) + rows * cols * bitdepth / 8); } else { n_frames = 0; } @@ -42,7 +42,7 @@ size_t SubFile::get_part(std::byte *buffer, size_t frame_index) { if (frame_index >= n_frames) { throw std::runtime_error("Frame number out of range"); } - fseek(fp, sizeof(sls_detector_header) + (sizeof(sls_detector_header) + bytes_per_part()) * frame_index, // NOLINT + fseek(fp, sizeof(DetectorHeader) + (sizeof(DetectorHeader) + bytes_per_part()) * frame_index, // NOLINT SEEK_SET); if (pixel_map){ @@ -63,11 +63,11 @@ size_t SubFile::get_part(std::byte *buffer, size_t frame_index) { } } -size_t SubFile::write_part(std::byte *buffer, sls_detector_header header, size_t frame_index) { +size_t SubFile::write_part(std::byte *buffer, DetectorHeader header, size_t frame_index) { if (frame_index > n_frames) { throw std::runtime_error("Frame number out of range"); } - fseek(fp, static_cast((sizeof(sls_detector_header) + bytes_per_part()) * frame_index), SEEK_SET); + fseek(fp, static_cast((sizeof(DetectorHeader) + bytes_per_part()) * frame_index), SEEK_SET); auto wc = fwrite(reinterpret_cast(&header), sizeof(header), 1, fp); wc += fwrite(buffer, bytes_per_part(), 1, fp); @@ -75,8 +75,8 @@ size_t SubFile::write_part(std::byte *buffer, sls_detector_header header, size_t } size_t SubFile::frame_number(size_t frame_index) { - sls_detector_header h{}; - fseek(fp, (sizeof(sls_detector_header) + bytes_per_part()) * frame_index, SEEK_SET); // NOLINT + DetectorHeader h{}; + fseek(fp, (sizeof(DetectorHeader) + bytes_per_part()) * frame_index, SEEK_SET); // NOLINT size_t const rc = fread(reinterpret_cast(&h), sizeof(h), 1, fp); if (rc != 1) throw std::runtime_error(LOCATION + "Could not read header from file"); diff --git a/src/defs.cpp b/src/defs.cpp index 27332a23..9cb599fd 100644 --- a/src/defs.cpp +++ b/src/defs.cpp @@ -68,6 +68,14 @@ template <> TimingMode StringTo(const std::string &arg) { throw std::runtime_error("Could not decode timing mode from: \"" + arg + "\""); } +template <> FrameDiscardPolicy StringTo(const std::string &arg) { + if (arg == "nodiscard") + return FrameDiscardPolicy::NoDiscard; + if (arg == "discard") + return FrameDiscardPolicy::Discard; + throw std::runtime_error("Could not decode frame discard policy from: \"" + arg + "\""); +} + // template <> TimingMode StringTo(std::string mode); } // namespace aare \ No newline at end of file diff --git a/src/file_utils.cpp b/src/file_utils.cpp deleted file mode 100644 index 01f0e441..00000000 --- a/src/file_utils.cpp +++ /dev/null @@ -1,33 +0,0 @@ -#include "aare/file_utils.hpp" - -namespace aare { - - - -bool is_master_file(const std::filesystem::path &fpath) { - std::string const stem = fpath.stem().string(); - return stem.find("_master_") != std::string::npos; -} - -FileNameComponents parse_fname(const std::filesystem::path &fname) { - FileNameComponents fnc; - fnc.base_path = fname.parent_path(); - fnc.base_name = fname.stem(); - fnc.ext = fname.extension(); - try { - auto pos = fnc.base_name.rfind('_'); - fnc.findex = std::stoi(fnc.base_name.substr(pos + 1)); - } catch (const std::invalid_argument &e) { - fnc.valid = false; - } - auto pos = fnc.base_name.find("_master_"); - if (pos != std::string::npos) { - fnc.base_name.erase(pos); - }else{ - fnc.valid = false; - } - fnc.valid = true; - return fnc; -} - -} // namespace aare \ No newline at end of file diff --git a/src/file_utils.test.cpp b/src/file_utils.test.cpp deleted file mode 100644 index e46ac3d8..00000000 --- a/src/file_utils.test.cpp +++ /dev/null @@ -1,20 +0,0 @@ -#include "aare/file_utils.hpp" - -#include - -using namespace aare; - -TEST_CASE("Use filename to determine if it is a master file") { - - REQUIRE(is_master_file("test_master_1.json")); -} - -TEST_CASE("Parse a master file fname"){ - auto fnc = parse_fname("test_master_1.json"); - REQUIRE(fnc.base_name == "test"); - REQUIRE(fnc.ext == ".json"); - REQUIRE(fnc.findex == 1); - - REQUIRE(fnc.master_fname() == "test_master_1.json"); - REQUIRE(fnc.data_fname(1, 2) == "test_d2_f1_1.raw"); -} \ No newline at end of file