Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'developer' into feature/github-ci2
Browse files Browse the repository at this point in the history
Bechir committed Apr 2, 2024
2 parents fe29600 + f6e2d35 commit 0ad4d31
Showing 7 changed files with 126 additions and 174 deletions.
56 changes: 0 additions & 56 deletions file_io/include/aare/FileInterface.h

This file was deleted.

2 changes: 1 addition & 1 deletion file_io/include/aare/FileInterface.hpp
Original file line number Diff line number Diff line change
@@ -83,7 +83,7 @@ class FileInterface {

public:
std::string mode;
std::filesystem::path m_fname;
// std::filesystem::path m_fname;
std::filesystem::path m_base_path;
std::string m_base_name, m_ext;
int m_findex;
38 changes: 20 additions & 18 deletions file_io/include/aare/NumpyFile.hpp
Original file line number Diff line number Diff line change
@@ -4,15 +4,31 @@
#include "aare/defs.hpp"
#include <iostream>
#include <numeric>
#include <filesystem>

namespace aare{

class NumpyFile : public FileInterface {
FILE *fp = nullptr;
size_t initial_header_len = 0;
size_t current_frame{};
std::filesystem::path m_fname;
uint32_t header_len{};
uint8_t header_len_size{};
ssize_t header_size{};
header_t m_header;
uint8_t major_ver_{};
uint8_t minor_ver_{};

void load_metadata();
void get_frame_into(size_t, std::byte *);
Frame get_frame(size_t frame_number);

public:


NumpyFile(const std::filesystem::path& fname);
NumpyFile(FileConfig, header_t);
void write(Frame &frame) override;
Frame read() override { return get_frame(this->current_frame++); }

@@ -24,26 +40,12 @@ class NumpyFile : public FileInterface {
size_t pixels() override;
void seek(size_t frame_number) override { this->current_frame = frame_number; }
size_t tell() override { return this->current_frame; }
size_t total_frames() const override { return header.shape[0]; }
ssize_t rows() const override { return header.shape[1]; }
ssize_t cols() const override { return header.shape[2]; }
ssize_t bitdepth() const override { return header.dtype.bitdepth(); }

NumpyFile(std::filesystem::path fname);
NumpyFile(FileConfig, header_t);
header_t header;
uint8_t major_ver_{};
uint8_t minor_ver_{};
uint32_t header_len{};
uint8_t header_len_size{};
ssize_t header_size{};
size_t total_frames() const override { return m_header.shape[0]; }
ssize_t rows() const override { return m_header.shape[1]; }
ssize_t cols() const override { return m_header.shape[2]; }
ssize_t bitdepth() const override { return m_header.dtype.bitdepth(); }

~NumpyFile();

private:
size_t current_frame{};
void get_frame_into(size_t, std::byte *);
Frame get_frame(size_t frame_number);
};

} // namespace aare
2 changes: 1 addition & 1 deletion file_io/include/aare/NumpyFileFactory.hpp
Original file line number Diff line number Diff line change
@@ -13,7 +13,7 @@ class NumpyFileFactory : public FileFactory {

public:
NumpyFileFactory(std::filesystem::path fpath);
void parse_metadata(FileInterface *_file) override;
void parse_metadata(FileInterface *_file) override{/*TODO! remove after refactor*/};
NumpyFile* load_file_read() override;
NumpyFile* load_file_write(FileConfig) override;
void parse_fname(FileInterface*)override{};
2 changes: 2 additions & 0 deletions file_io/include/aare/RawFile.hpp
Original file line number Diff line number Diff line change
@@ -9,8 +9,10 @@ namespace aare {
class RawFile : public FileInterface {

using config = RawFileConfig;


public:
std::filesystem::path m_fname; //TO be made private!
void write(Frame &frame) override{};
Frame read() override { return get_frame(this->current_frame++); };
std::vector<Frame> read(size_t n_frames) override;
132 changes: 102 additions & 30 deletions file_io/src/NumpyFile.cpp
Original file line number Diff line number Diff line change
@@ -3,6 +3,33 @@

namespace aare{

NumpyFile::NumpyFile(const std::filesystem::path& fname) {
//TODO! add opts to constructor
m_fname = fname;
fp = fopen(m_fname.c_str(), "rb");
if (!fp) {
throw std::runtime_error(fmt::format("Could not open: {} for reading", m_fname.c_str()));
}
load_metadata();
}
NumpyFile::NumpyFile(FileConfig config, header_t header) {
mode = "w";
m_fname = config.fname;
m_bitdepth = config.dtype.bitdepth();
m_rows = config.rows;
m_cols = config.cols;
m_header = header;
m_header.shape = {0, config.rows, config.cols};

fp = fopen(m_fname.c_str(), "wb");
if (!fp) {
throw std::runtime_error(fmt::format("Could not open: {} for reading", m_fname.c_str()));
}

initial_header_len =
aare::NumpyHelpers::write_header(std::filesystem::path(m_fname.c_str()), header);
}

void NumpyFile::write(Frame &frame) {
if (fp == nullptr) {
throw std::runtime_error("File not open");
@@ -14,63 +41,43 @@ void NumpyFile::write(Frame &frame) {
fwrite(frame._get_data(), frame.size(), 1, fp);
}

NumpyFile::NumpyFile(std::filesystem::path fname_) {
this->m_fname = fname_;
fp = fopen(this->m_fname.c_str(), "rb");
}
NumpyFile::NumpyFile(FileConfig config, header_t header) {
this->mode = "w";
this->m_fname = config.fname;
this->m_bitdepth = config.dtype.bitdepth();
this->m_rows = config.rows;
this->m_cols = config.cols;
this->header = header;
this->header.shape = {0, config.rows, config.cols};

fp = fopen(this->m_fname.c_str(), "wb");
if (!fp) {
throw std::runtime_error(fmt::format("Could not open: {} for reading", this->m_fname.c_str()));
}

this->initial_header_len =
aare::NumpyHelpers::write_header(std::filesystem::path(this->m_fname.c_str()), this->header);
}

Frame NumpyFile::get_frame(size_t frame_number) {
Frame frame(header.shape[1], header.shape[2], header.dtype.bitdepth());
Frame frame(m_header.shape[1], m_header.shape[2], m_header.dtype.bitdepth());
get_frame_into(frame_number, frame._get_data());
return frame;
}
void NumpyFile::get_frame_into(size_t frame_number, std::byte *image_buf) {
if (fp == nullptr) {
throw std::runtime_error("File not open");
}
if (frame_number > header.shape[0]) {
if (frame_number > m_header.shape[0]) {
throw std::runtime_error("Frame number out of range");
}
fseek(fp, header_size + frame_number * bytes_per_frame(), SEEK_SET);
fread(image_buf, bytes_per_frame(), 1, fp);
}

size_t NumpyFile::pixels() {
return std::accumulate(header.shape.begin() + 1, header.shape.end(), 1, std::multiplies<uint64_t>());
return std::accumulate(m_header.shape.begin() + 1, m_header.shape.end(), 1, std::multiplies<uint64_t>());
};
size_t NumpyFile::bytes_per_frame() { return header.dtype.bitdepth() / 8 * pixels(); };
size_t NumpyFile::bytes_per_frame() { return m_header.dtype.bitdepth() / 8 * pixels(); };

std::vector<Frame> NumpyFile::read(size_t n_frames) {
// TODO: implement this in a more efficient way
std::vector<Frame> frames;
for (size_t i = 0; i < n_frames; i++) {
frames.push_back(this->get_frame(this->current_frame));
this->current_frame++;
frames.push_back(get_frame(current_frame));
current_frame++;
}
return frames;
}
void NumpyFile::read_into(std::byte *image_buf, size_t n_frames) {
// TODO: implement this in a more efficient way
for (size_t i = 0; i < n_frames; i++) {
this->get_frame_into(this->current_frame++, image_buf);
image_buf += this->bytes_per_frame();
get_frame_into(current_frame++, image_buf);
image_buf += bytes_per_frame();
}
}

@@ -82,11 +89,11 @@ NumpyFile::~NumpyFile() {
size_t data_size = file_size - initial_header_len;
size_t n_frames = data_size / bytes_per_frame();
// update number of frames in header (first element of shape)
this->header.shape[0] = n_frames;
m_header.shape[0] = n_frames;
fseek(fp, 0, SEEK_SET);
// create string stream to contain header
std::stringstream ss;
aare::NumpyHelpers::write_header(ss, this->header);
aare::NumpyHelpers::write_header(ss, m_header);
std::string header_str = ss.str();
// write header
fwrite(header_str.c_str(), header_str.size(), 1, fp);
@@ -100,4 +107,69 @@ NumpyFile::~NumpyFile() {
}
}

void NumpyFile::load_metadata(){

// read magic number
std::array<char, 6> tmp{};
fread(tmp.data(), tmp.size(), 1, fp);
if (tmp != aare::NumpyHelpers::magic_str) {
for (auto item : tmp)
fmt::print("{}, ", int(item));
fmt::print("\n");
throw std::runtime_error("Not a numpy file");
}

// read version
fread(reinterpret_cast<char *>(&major_ver_), sizeof(major_ver_), 1,fp);
fread(reinterpret_cast<char *>(&minor_ver_), sizeof(minor_ver_), 1,fp);

if (major_ver_ == 1) {
header_len_size = 2;
} else if (major_ver_ == 2) {
header_len_size = 4;
} else {
throw std::runtime_error("Unsupported numpy version");
}

// read header length
fread(reinterpret_cast<char *>(&header_len), header_len_size,1, fp);
header_size = aare::NumpyHelpers::magic_string_length + 2 + header_len_size + header_len;
if (header_size % 16 != 0) {
fmt::print("Warning: header length is not a multiple of 16\n");
}

// read header
auto buf_v = std::vector<char>(header_len);
fread(buf_v.data(), header_len,1,fp);
std::string header(buf_v.data(), header_len);

// parse header
std::vector<std::string> keys{"descr", "fortran_order", "shape"};
aare::logger::debug("original header: \"header\"");

auto dict_map = aare::NumpyHelpers::parse_dict(header, keys);
if (dict_map.size() == 0)
throw std::runtime_error("invalid dictionary in header");

std::string descr_s = dict_map["descr"];
std::string fortran_s = dict_map["fortran_order"];
std::string shape_s = dict_map["shape"];

std::string descr = aare::NumpyHelpers::parse_str(descr_s);
aare::DType dtype = aare::NumpyHelpers::parse_descr(descr);

// convert literal Python bool to C++ bool
bool fortran_order = aare::NumpyHelpers::parse_bool(fortran_s);

// parse the shape tuple
auto shape_v = aare::NumpyHelpers::parse_tuple(shape_s);
shape_t shape;
for (auto item : shape_v) {
auto dim = static_cast<unsigned long>(std::stoul(item));
shape.push_back(dim);
}
m_header = {dtype, fortran_order, shape};
}


} // namespace aare
68 changes: 0 additions & 68 deletions file_io/src/NumpyFileFactory.cpp
Original file line number Diff line number Diff line change
@@ -4,78 +4,10 @@
namespace aare {

NumpyFileFactory::NumpyFileFactory(std::filesystem::path fpath) { this->m_fpath = fpath; }
void NumpyFileFactory::parse_metadata(FileInterface *_file) {
auto file = dynamic_cast<NumpyFile *>(_file);
// open ifsteam to file
f = std::ifstream(file->m_fname, std::ios::binary);
// check if file exists
if (!f.is_open()) {
throw std::runtime_error(fmt::format("Could not open: {} for reading", file->m_fname.c_str()));
}
// read magic number
std::array<char, 6> tmp{};
f.read(tmp.data(), tmp.size());
if (tmp != aare::NumpyHelpers::magic_str) {
for (auto item : tmp)
fmt::print("{}, ", int(item));
fmt::print("\n");
throw std::runtime_error("Not a numpy file");
}

// read version
f.read(reinterpret_cast<char *>(&file->major_ver_), 1);
f.read(reinterpret_cast<char *>(&file->minor_ver_), 1);

if (file->major_ver_ == 1) {
file->header_len_size = 2;
} else if (file->major_ver_ == 2) {
file->header_len_size = 4;
} else {
throw std::runtime_error("Unsupported numpy version");
}
// read header length
f.read(reinterpret_cast<char *>(&file->header_len), file->header_len_size);
file->header_size = aare::NumpyHelpers::magic_string_length + 2 + file->header_len_size + file->header_len;
if (file->header_size % 16 != 0) {
fmt::print("Warning: header length is not a multiple of 16\n");
}
// read header
auto buf_v = std::vector<char>(file->header_len);
f.read(buf_v.data(), file->header_len);
std::string header(buf_v.data(), file->header_len);

// parse header

std::vector<std::string> keys{"descr", "fortran_order", "shape"};
aare::logger::debug("original header: \"header\"");

auto dict_map = aare::NumpyHelpers::parse_dict(header, keys);
if (dict_map.size() == 0)
throw std::runtime_error("invalid dictionary in header");

std::string descr_s = dict_map["descr"];
std::string fortran_s = dict_map["fortran_order"];
std::string shape_s = dict_map["shape"];

std::string descr = aare::NumpyHelpers::parse_str(descr_s);
aare::DType dtype = aare::NumpyHelpers::parse_descr(descr);

// convert literal Python bool to C++ bool
bool fortran_order = aare::NumpyHelpers::parse_bool(fortran_s);

// parse the shape tuple
auto shape_v = aare::NumpyHelpers::parse_tuple(shape_s);
shape_t shape;
for (auto item : shape_v) {
auto dim = static_cast<unsigned long>(std::stoul(item));
shape.push_back(dim);
}
file->header = {dtype, fortran_order, shape};
}

NumpyFile *NumpyFileFactory::load_file_read() {
NumpyFile *file = new NumpyFile(this->m_fpath);
parse_metadata(file);
return file;
};

0 comments on commit 0ad4d31

Please sign in to comment.