added CtbRawFile

CMakeLists.txt

@@ -237,7 +237,7 @@ set(PUBLICHEADERS
     include/aare/Dtype.hpp
     include/aare/File.hpp
     include/aare/FileInterface.hpp
-    include/aare/file_utils.hpp
+    include/aare/RawMasterFile.hpp
     include/aare/Frame.hpp
     include/aare/json.hpp
     include/aare/NDArray.hpp
@@ -259,7 +259,7 @@ set(SourceFiles
     ${CMAKE_CURRENT_SOURCE_DIR}/src/Dtype.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/Frame.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/File.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/src/file_utils.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/RawMasterFile.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/NumpyFile.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/PixelMap.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/RawFile.cpp
@@ -290,7 +290,7 @@ if(AARE_TESTS)
             ${CMAKE_CURRENT_SOURCE_DIR}/src/defs.test.cpp
             ${CMAKE_CURRENT_SOURCE_DIR}/src/Dtype.test.cpp
             ${CMAKE_CURRENT_SOURCE_DIR}/src/Frame.test.cpp
-            ${CMAKE_CURRENT_SOURCE_DIR}/src/file_utils.test.cpp
+            ${CMAKE_CURRENT_SOURCE_DIR}/src/RawMasterFile.test.cpp
             ${CMAKE_CURRENT_SOURCE_DIR}/src/NDArray.test.cpp
             ${CMAKE_CURRENT_SOURCE_DIR}/src/NDView.test.cpp
             ${CMAKE_CURRENT_SOURCE_DIR}/src/ClusterFinder.test.cpp

docs/CMakeLists.txt

@@ -22,6 +22,7 @@ set(SPHINX_SOURCE_FILES
   src/VarClusterFinder.rst
   src/pyVarClusterFinder.rst
   src/pyFile.rst
+  src/pyCtbRawFile.rst
 )
 
 foreach(filename ${SPHINX_SOURCE_FILES})

docs/src/index.rst

@@ -23,4 +23,6 @@ AARE
     :maxdepth: 1
 
     pyFile
+    pyCtbRawFile
     pyVarClusterFinder
+

docs/src/pyCtbRawFile.rst

@@ -0,0 +1,11 @@
+
+CtbRawFile
+============
+
+.. py:currentmodule:: aare
+
+.. autoclass:: CtbRawFile
+    :members:
+    :undoc-members:
+    :show-inheritance:
+    :inherited-members:

include/aare/CtbRawFile.hpp

@@ -1,19 +1,37 @@
 #pragma once
 
 #include "aare/FileInterface.hpp"
-#include "aare/file_utils.hpp"
+#include "aare/RawMasterFile.hpp"
 #include "aare/Frame.hpp"
 
 #include <filesystem>
+#include <fstream>
 
 namespace aare{
 
 class CtbRawFile{
-    FileNameComponents m_fnc{};
+    RawMasterFile m_master;
+    std::ifstream m_file;
+    size_t m_current_frame{0};
+    size_t m_current_subfile{0};
+    size_t m_num_subfiles{0};
 public:
     CtbRawFile(const std::filesystem::path &fname);
 
+    void read_into(std::byte *image_buf, DetectorHeader* header = nullptr);
+    void seek(size_t frame_index);          //!< seek to the given frame index
+    size_t tell() const;                    //!< get the frame index of the file pointer
 
+    // in the specific class we can expose more functionality
+
+    size_t image_size_in_bytes() const { return m_master.image_size_in_bytes(); }
+
+private:
+    void find_subfiles();
+    size_t sub_file_index(size_t frame_index) const {
+        return frame_index / m_master.max_frames_per_file();
+    }
+    void open_data_file(size_t subfile_index);
 
 };
 

include/aare/RawFile.hpp

@@ -180,9 +180,9 @@ class RawFile : public FileInterface {
     /**
      * @brief read the header of the file
      * @param fname path to the data subfile
-     * @return sls_detector_header
+     * @return DetectorHeader
      */
-    static sls_detector_header read_header(const std::filesystem::path &fname);
+    static DetectorHeader read_header(const std::filesystem::path &fname);
 
     /**
      * @brief open the subfiles

include/aare/RawMasterFile.hpp

@@ -0,0 +1,96 @@
+#pragma once
+#include "aare/defs.hpp"
+#include <filesystem>
+#include <fmt/format.h>
+#include <fstream>
+
+#include <nlohmann/json.hpp>
+using json = nlohmann::json;
+
+namespace aare {
+
+class RawFileNameComponents {
+    std::filesystem::path m_base_path{};
+    std::string m_base_name{};
+    std::string m_ext{};
+    int m_file_index{}; // TODO! is this measurement_index?
+
+  public:
+    RawFileNameComponents(const std::filesystem::path &fname);
+    std::filesystem::path master_fname() const {
+        return m_base_path /
+               fmt::format("{}_master_{}{}", m_base_name, m_file_index, m_ext);
+    }
+
+    std::filesystem::path data_fname(size_t mod_id, size_t file_id) {
+        return m_base_path / fmt::format("{}_d{}_f{}_{}.raw", m_base_name,
+                                         mod_id, file_id, m_file_index);
+    }
+
+    const std::filesystem::path &base_path() const { return m_base_path; }
+    const std::string &base_name() const { return m_base_name; }
+    const std::string &ext() const { return m_ext; }
+    int file_index() const { return m_file_index; }
+};
+
+class RawMasterFile {
+    RawFileNameComponents m_fnc;
+    std::string m_version;
+    DetectorType m_type;
+    TimingMode m_timing_mode;
+
+    size_t m_image_size_in_bytes;
+    size_t m_frames_in_file;
+    size_t m_pixels_y;
+    size_t m_pixels_x;
+    size_t m_bitdepth;
+
+    size_t m_max_frames_per_file;
+    uint32_t m_adc_mask;
+    FrameDiscardPolicy m_frame_discard_policy;
+    size_t m_frame_padding;
+
+    std::optional<size_t> m_analog_samples;
+    std::optional<size_t> m_digital_samples;
+
+  public:
+    RawMasterFile(const std::filesystem::path &fpath) : m_fnc(fpath) {
+        if (!std::filesystem::exists(fpath)) {
+            throw std::runtime_error(LOCATION + " File does not exist");
+        }
+        if (m_fnc.ext() == ".json") {
+            parse_json(fpath);
+        } else if (m_fnc.ext() == ".raw") {
+            parse_raw(fpath);
+        } else {
+            throw std::runtime_error(LOCATION + "Unsupported file type");
+        }
+    }
+
+    const std::string &version() const { return m_version; }
+    const DetectorType &detector_type() const { return m_type; }
+    const TimingMode &timing_mode() const { return m_timing_mode; }
+    size_t image_size_in_bytes() const { return m_image_size_in_bytes; }
+    size_t frames_in_file() const { return m_frames_in_file; }
+    size_t pixels_y() const { return m_pixels_y; }
+    size_t pixels_x() const { return m_pixels_x; }
+    size_t max_frames_per_file() const { return m_max_frames_per_file; }
+    size_t bitdepth() const { return m_bitdepth; }
+    size_t frame_padding() const { return m_frame_padding; }
+    const FrameDiscardPolicy &frame_discard_policy() const {
+        return m_frame_discard_policy;
+    }
+
+    std::optional<size_t> analog_samples() const { return m_analog_samples; }
+    std::optional<size_t> digital_samples() const { return m_digital_samples; }
+
+    std::filesystem::path data_fname(size_t mod_id, size_t file_id) {
+        return m_fnc.data_fname(mod_id, file_id);
+    }
+
+  private:
+    void parse_json(const std::filesystem::path &fpath);
+    void parse_raw(const std::filesystem::path &fpath);
+};
+
+} // namespace aare

include/aare/SubFile.hpp

@@ -15,7 +15,7 @@ namespace aare {
  */
 class SubFile {
   public:
-    size_t write_part(std::byte *buffer, sls_detector_header header, size_t frame_index);
+    size_t write_part(std::byte *buffer, DetectorHeader header, size_t frame_index);
     /**
      * @brief SubFile constructor
      * @param fname path to the subfile

include/aare/defs.hpp

@@ -111,7 +111,7 @@ class Cluster {
 /**
  * @brief header contained in parts of frames
  */
-struct sls_detector_header {
+struct DetectorHeader {
     uint64_t frameNumber;
     uint32_t expLength;
     uint32_t packetNumber;
@@ -166,6 +166,7 @@ using xy = t_xy<uint32_t>;
 
 using dynamic_shape = std::vector<int64_t>;
 
+//TODO! Can we uniform enums between the libraries?
 enum class DetectorType {
     Jungfrau,
     Eiger,
@@ -178,6 +179,7 @@ enum class DetectorType {
 };
 
 enum class TimingMode { Auto, Trigger };
+enum class FrameDiscardPolicy { NoDiscard, Discard };
 
 template <class T> T StringTo(const std::string &arg) { return T(arg); }
 
@@ -188,6 +190,8 @@ template <> std::string ToString(DetectorType arg);
 
 template <> TimingMode StringTo(const std::string & /*mode*/);
 
+template <> FrameDiscardPolicy StringTo(const std::string & /*mode*/);
+
 using DataTypeVariants = std::variant<uint16_t, uint32_t>;
 
 } // namespace aare

python/CMakeLists.txt

@@ -27,6 +27,8 @@ target_link_libraries(_aare PRIVATE aare_core aare_compiler_flags)
 # List of python files to be copied to the build directory
 set( PYTHON_FILES
     aare/__init__.py
+    aare/CtbRawFile.py
+    aare/transform.py
 )
 
 # Copy the python files to the build directory