From 67cf3ec9e0616f7abe6ee5014ec89b681e3ef414 Mon Sep 17 00:00:00 2001 From: bkhalifaconti <110467891+bkhalifaconti@users.noreply.github.com> Date: Fri, 9 Sep 2022 12:29:16 +0200 Subject: [PATCH] eCAL Rec / ecalhdf5: Support one file per topic feature (#755) Co-authored-by: Florian Reimold <11774314+FlorianReimold@users.noreply.github.com> --- .../src/ecal/app/pb/rec/client_service.proto | 1 + .../src/ecal/app/pb/rec/server_config.proto | 1 + .../rec_client_cli/src/ecal_rec_service.cpp | 16 + .../include/rec_client_core/ecal_rec_defs.h | 2 +- .../include/rec_client_core/job_config.h | 4 + .../src/job/hdf5_writer_thread.cpp | 1 + app/rec/rec_client_core/src/job_config.cpp | 20 +- app/rec/rec_gui/src/qecalrec.cpp | 9 +- app/rec/rec_gui/src/qecalrec.h | 18 +- .../widgets/config_widget/config_widget.cpp | 17 +- .../src/widgets/config_widget/config_widget.h | 1 + .../widgets/config_widget/config_widget.ui | 151 +++++-- .../src/commands/set_config.cpp | 15 + .../rec_server_cli/src/commands/set_config.h | 4 + .../src/ecal_rec_server_cli.cpp | 35 ++ app/rec/rec_server_core/CMakeLists.txt | 4 +- .../include/rec_server_core/rec_server.h | 10 +- .../rec_server_core/rec_server_config.h | 4 +- app/rec/rec_server_core/src/config/config.cpp | 67 ++- app/rec/rec_server_core/src/config/config.h | 2 +- .../{config_v2to3.cpp => config_v2to4.cpp} | 36 +- .../config/{config_v2to3.h => config_v2to4.h} | 5 +- app/rec/rec_server_core/src/proto_helpers.cpp | 10 +- app/rec/rec_server_core/src/rec_server.cpp | 12 +- .../rec_server_core/src/rec_server_impl.cpp | 11 + app/rec/rec_server_core/src/rec_server_impl.h | 20 +- .../src/recorder/remote_recorder.cpp | 11 +- contrib/ecalhdf5/CMakeLists.txt | 2 + contrib/ecalhdf5/include/ecalhdf5/eh5_meas.h | 22 + contrib/ecalhdf5/src/eh5_meas.cpp | 31 +- contrib/ecalhdf5/src/eh5_meas_dir.cpp | 330 +++++---------- contrib/ecalhdf5/src/eh5_meas_dir.h | 114 +++--- contrib/ecalhdf5/src/eh5_meas_file_v1.cpp | 39 +- contrib/ecalhdf5/src/eh5_meas_file_v1.h | 29 +- contrib/ecalhdf5/src/eh5_meas_file_v2.cpp | 28 +- contrib/ecalhdf5/src/eh5_meas_file_v2.h | 27 +- contrib/ecalhdf5/src/eh5_meas_file_v3.cpp | 6 +- contrib/ecalhdf5/src/eh5_meas_file_v3.h | 4 +- contrib/ecalhdf5/src/eh5_meas_file_v4.cpp | 6 +- contrib/ecalhdf5/src/eh5_meas_file_v4.h | 4 +- contrib/ecalhdf5/src/eh5_meas_file_v5.cpp | 6 +- contrib/ecalhdf5/src/eh5_meas_file_v5.h | 4 +- .../ecalhdf5/src/eh5_meas_file_writer_v5.cpp | 387 ++++++++++++++++++ .../ecalhdf5/src/eh5_meas_file_writer_v5.h | 344 ++++++++++++++++ contrib/ecalhdf5/src/eh5_meas_impl.h | 25 +- contrib/ecalhdf5/src/escape.cpp | 12 +- .../rec/rec_server_cli_interactive_help.txt | 1 + .../applications/rec/rec_server_cli_usage.txt | 6 + .../src/ecalrecorder_client.cpp | 12 +- .../src/EcalrecGuiClient.cpp | 5 + .../rec_client_service_gui/src/MainWindow.ui | 80 ++-- .../ecalhdf5/hdf5_test/src/hdf5_test.cpp | 138 ++++++- 52 files changed, 1648 insertions(+), 501 deletions(-) rename app/rec/rec_server_core/src/config/{config_v2to3.cpp => config_v2to4.cpp} (95%) rename app/rec/rec_server_core/src/config/{config_v2to3.h => config_v2to4.h} (95%) create mode 100644 contrib/ecalhdf5/src/eh5_meas_file_writer_v5.cpp create mode 100644 contrib/ecalhdf5/src/eh5_meas_file_writer_v5.h diff --git a/app/app_pb/src/ecal/app/pb/rec/client_service.proto b/app/app_pb/src/ecal/app/pb/rec/client_service.proto index 5ddcf1ea39..98e2b5245f 100644 --- a/app/app_pb/src/ecal/app/pb/rec/client_service.proto +++ b/app/app_pb/src/ecal/app/pb/rec/client_service.proto @@ -54,6 +54,7 @@ message Configuration // meas_name [string] The name of the measurement (un-evaluated format) // description [string] The description that will be saved to the measurement's doc folder (un-evaluated format) // max_file_size_mib [uint] The maximum HDF5 file size (When exceeding the file size, the measurement will be splitted into multiple files). + // one_file_per_topic [bool] Whether the recorder shall create 1 hdf5 file per channel // ==== Upload measurement config ==== // protocol [string] The upload type to use (e.g. ftp). More types may be added in the future, if necessary. diff --git a/app/app_pb/src/ecal/app/pb/rec/server_config.proto b/app/app_pb/src/ecal/app/pb/rec/server_config.proto index d0bb8e095c..5bb15d8aa2 100644 --- a/app/app_pb/src/ecal/app/pb/rec/server_config.proto +++ b/app/app_pb/src/ecal/app/pb/rec/server_config.proto @@ -57,6 +57,7 @@ message RecServerConfig string root_dir = 1; // The root directory where to save the measurements to string meas_name = 2; // The name for new measurements. This name will be used for the name of a sub-directory in the root dir. int64 max_file_size_mib = 3; // Max file size in MiB for HDF5 files + bool one_file_per_topic = 11; // Whether to create 1 HDF5 file per eCAL Topic string description = 4; // The description for new measurements. Will be stored in the /doc/direction.txt file. map enabled_clients_config = 5; // All enabled rec clients along with their specific configuration bool pre_buffer_enabled = 6; // Whether pre-buffering is enabled diff --git a/app/rec/rec_client_cli/src/ecal_rec_service.cpp b/app/rec/rec_client_cli/src/ecal_rec_service.cpp index dab00fd441..fa96159a9a 100644 --- a/app/rec/rec_client_cli/src/ecal_rec_service.cpp +++ b/app/rec/rec_client_cli/src/ecal_rec_service.cpp @@ -620,6 +620,22 @@ ::eCAL::rec::JobConfig EcalRecService::ToJobConfig(const ::eCAL::pb::rec_client: } } + ////////////////////////////////////// + // one_file_per_topic // + ////////////////////////////////////// + { + auto it = config.items().find("one_file_per_topic"); + if (it != config.items().end()) + { + std::string one_file_per_topic = it->second; + job_config.SetOneFilePerTopicEnabled(strToBool(one_file_per_topic)); + } + else + { + job_config.SetOneFilePerTopicEnabled(false); + } + } + ////////////////////////////////////// // description // ////////////////////////////////////// diff --git a/app/rec/rec_client_core/include/rec_client_core/ecal_rec_defs.h b/app/rec/rec_client_core/include/rec_client_core/ecal_rec_defs.h index b15fad06c1..e8de200b9d 100644 --- a/app/rec/rec_client_core/include/rec_client_core/ecal_rec_defs.h +++ b/app/rec/rec_client_core/include/rec_client_core/ecal_rec_defs.h @@ -20,7 +20,7 @@ #pragma once #define ECAL_REC_VERSION_MAJOR 2 -#define ECAL_REC_VERSION_MINOR 5 +#define ECAL_REC_VERSION_MINOR 6 #define ECAL_REC_VERSION_PATCH 0 #define __ECAL_REC_FUNCTION_TO_STR(x) #x // Stringify any input diff --git a/app/rec/rec_client_core/include/rec_client_core/job_config.h b/app/rec/rec_client_core/include/rec_client_core/job_config.h index 06cb34afb9..9eacb8bc82 100644 --- a/app/rec/rec_client_core/include/rec_client_core/job_config.h +++ b/app/rec/rec_client_core/include/rec_client_core/job_config.h @@ -52,6 +52,9 @@ namespace eCAL void SetMaxFileSize(int64_t max_file_size_mb); int64_t GetMaxFileSize() const; + void SetOneFilePerTopicEnabled(bool enabled); + bool GetOneFilePerTopicEnabled() const; + void SetDescription(const std::string& description); std::string GetDescription() const; @@ -72,6 +75,7 @@ namespace eCAL std::string meas_root_dir_; std::string meas_name_; int64_t max_file_size_mb_; + bool one_file_per_topic_; std::string description_; }; } diff --git a/app/rec/rec_client_core/src/job/hdf5_writer_thread.cpp b/app/rec/rec_client_core/src/job/hdf5_writer_thread.cpp index 2bdc56eac0..87212e5f4c 100644 --- a/app/rec/rec_client_core/src/job/hdf5_writer_thread.cpp +++ b/app/rec/rec_client_core/src/job/hdf5_writer_thread.cpp @@ -272,6 +272,7 @@ namespace eCAL hdf5_writer_->SetFileBaseName(host_name); hdf5_writer_->SetMaxSizePerFile(job_config_.GetMaxFileSize()); + hdf5_writer_->SetOneFilePerChannelEnabled(job_config_.GetOneFilePerTopicEnabled()); } else { diff --git a/app/rec/rec_client_core/src/job_config.cpp b/app/rec/rec_client_core/src/job_config.cpp index 99d4eb166a..f0a789d2ca 100644 --- a/app/rec/rec_client_core/src/job_config.cpp +++ b/app/rec/rec_client_core/src/job_config.cpp @@ -33,6 +33,7 @@ namespace eCAL JobConfig::JobConfig() : job_id_(0) , max_file_size_mb_(50) + , one_file_per_topic_(false) {} JobConfig::~JobConfig() @@ -62,17 +63,20 @@ namespace eCAL job_id_ = (((int64_t)counter) << 32) | (int64_t(0xFFFFF000) & system_clock_nsecs) | (int64_t(0x00000FFF) & steady_clock_nsecs); } - void JobConfig::SetMeasRootDir (const std::string& meas_root_dir) { meas_root_dir_ = meas_root_dir; } - std::string JobConfig::GetMeasRootDir () const { return meas_root_dir_; } + void JobConfig::SetMeasRootDir (const std::string& meas_root_dir) { meas_root_dir_ = meas_root_dir; } + std::string JobConfig::GetMeasRootDir () const { return meas_root_dir_; } - void JobConfig::SetMeasName (const std::string& meas_name) { meas_name_ = meas_name; } - std::string JobConfig::GetMeasName () const { return meas_name_; } + void JobConfig::SetMeasName (const std::string& meas_name) { meas_name_ = meas_name; } + std::string JobConfig::GetMeasName () const { return meas_name_; } - void JobConfig::SetMaxFileSize (int64_t max_file_size_mb) { max_file_size_mb_ = max_file_size_mb; } - int64_t JobConfig::GetMaxFileSize () const { return max_file_size_mb_; } + void JobConfig::SetMaxFileSize (int64_t max_file_size_mb) { max_file_size_mb_ = max_file_size_mb; } + int64_t JobConfig::GetMaxFileSize () const { return max_file_size_mb_; } - void JobConfig::SetDescription (const std::string& description) { description_ = description; } - std::string JobConfig::GetDescription () const { return description_; } + void JobConfig::SetOneFilePerTopicEnabled(bool enabled) { one_file_per_topic_ = enabled; } + bool JobConfig::GetOneFilePerTopicEnabled() const { return one_file_per_topic_; } + + void JobConfig::SetDescription (const std::string& description) { description_ = description; } + std::string JobConfig::GetDescription () const { return description_; } ////////////////////////////// // Evaluation diff --git a/app/rec/rec_gui/src/qecalrec.cpp b/app/rec/rec_gui/src/qecalrec.cpp index 1ffcd0d24b..af78d54dbb 100644 --- a/app/rec/rec_gui/src/qecalrec.cpp +++ b/app/rec/rec_gui/src/qecalrec.cpp @@ -560,6 +560,13 @@ void QEcalRec::setMaxFileSizeMib(unsigned int max_file_size_mib) emit maxFileSizeMibChangedSignal(max_file_size_mib); } +void QEcalRec::setOneFilePerTopicEnabled(bool enabled) +{ + updateConfigModified(true); + rec_server_->SetOneFilePerTopicEnabled(enabled); + emit oneFilePerTopicEnabledChangedSignal(enabled); +} + void QEcalRec::setDescription(const std::string& description) { updateConfigModified(true); @@ -922,7 +929,7 @@ bool QEcalRec::loadConfigFromFile(const std::string& path, bool omit_dialogs) emit measNameChangedSignal (rec_server_->GetMeasName()); emit maxFileSizeMibChangedSignal(rec_server_->GetMaxFileSizeMib()); emit descriptionChangedSignal (rec_server_->GetDescription()); - + emit oneFilePerTopicEnabledChangedSignal(rec_server_->GetOneFilePerTopicEnabled()); emit usingBuiltInRecorderEnabledChangedSignal(rec_server_->IsUsingBuiltInRecorderEnabled()); diff --git a/app/rec/rec_gui/src/qecalrec.h b/app/rec/rec_gui/src/qecalrec.h index 5f01af8294..b1ed634081 100644 --- a/app/rec/rec_gui/src/qecalrec.h +++ b/app/rec/rec_gui/src/qecalrec.h @@ -155,10 +155,11 @@ public slots: // Job Settings //////////////////////////////////// public slots: - void setMeasRootDir (const std::string& meas_root_dir); - void setMeasName (const std::string& meas_name); - void setMaxFileSizeMib (unsigned int max_file_size_mib); - void setDescription (const std::string& description); + void setMeasRootDir (const std::string& meas_root_dir); + void setMeasName (const std::string& meas_name); + void setMaxFileSizeMib (unsigned int max_file_size_mib); + void setOneFilePerTopicEnabled(bool enabled); + void setDescription (const std::string& description); std::string measRootDir() const; std::string measName() const; @@ -166,10 +167,11 @@ public slots: std::string description() const; signals: - void measRootDirChangedSignal (std::string meas_root_dir); - void measNameChangedSignal (std::string meas_name); - void maxFileSizeMibChangedSignal(unsigned int max_file_size_mib); - void descriptionChangedSignal (std::string description); + void measRootDirChangedSignal (std::string meas_root_dir); + void measNameChangedSignal (std::string meas_name); + void maxFileSizeMibChangedSignal (unsigned int max_file_size_mib); + void oneFilePerTopicEnabledChangedSignal(bool enabled); + void descriptionChangedSignal (std::string description); //////////////////////////////////// // Server Settings diff --git a/app/rec/rec_gui/src/widgets/config_widget/config_widget.cpp b/app/rec/rec_gui/src/widgets/config_widget/config_widget.cpp index 755600d7a2..7163f7cd40 100644 --- a/app/rec/rec_gui/src/widgets/config_widget/config_widget.cpp +++ b/app/rec/rec_gui/src/widgets/config_widget/config_widget.cpp @@ -30,10 +30,11 @@ ConfigWidget::ConfigWidget(QWidget *parent) { ui_.setupUi(this); - connect(ui_.measurement_directory_lineedit, &QLineEdit::textChanged, QEcalRec::instance(), [this]() {QEcalRec::instance()->setMeasRootDir(ui_.measurement_directory_lineedit->text().toStdString()); }); - connect(ui_.measurement_name_lineedit, &QLineEdit::textChanged, QEcalRec::instance(), [this]() {QEcalRec::instance()->setMeasName (ui_.measurement_name_lineedit->text().toStdString()); }); - connect(ui_.description_textedit, &QTextEdit::textChanged, QEcalRec::instance(), [this]() {QEcalRec::instance()->setDescription(ui_.description_textedit->toPlainText().toStdString()); }); + connect(ui_.measurement_directory_lineedit, &QLineEdit::textChanged, QEcalRec::instance(), [this]() {QEcalRec::instance()->setMeasRootDir(ui_.measurement_directory_lineedit->text().toStdString()); }); + connect(ui_.measurement_name_lineedit, &QLineEdit::textChanged, QEcalRec::instance(), [this]() {QEcalRec::instance()->setMeasName (ui_.measurement_name_lineedit->text().toStdString()); }); + connect(ui_.description_textedit, &QTextEdit::textChanged, QEcalRec::instance(), [this]() {QEcalRec::instance()->setDescription(ui_.description_textedit->toPlainText().toStdString()); }); connect(ui_.max_file_size_spinbox, static_cast(&QSpinBox::valueChanged), QEcalRec::instance(), [](int megabytes) {QEcalRec::instance()->setMaxFileSizeMib(megabytes); }); + connect(ui_.one_file_per_topic_checkbox, &QCheckBox::stateChanged, QEcalRec::instance(), [this]() {QEcalRec::instance()->setOneFilePerTopicEnabled(ui_.one_file_per_topic_checkbox->isChecked()); }); connect(ui_.refresh_path_preview_button, &QAbstractButton::clicked, QEcalRec::instance(), [this]() { updatePathPreviewAndWarningLabel(); }); @@ -41,6 +42,7 @@ ConfigWidget::ConfigWidget(QWidget *parent) connect(QEcalRec::instance(), &QEcalRec::measNameChangedSignal, this, &ConfigWidget::measurementNameChanged); connect(QEcalRec::instance(), &QEcalRec::maxFileSizeMibChangedSignal, this, &ConfigWidget::maxFileSizeChanged); connect(QEcalRec::instance(), &QEcalRec::descriptionChangedSignal, this, &ConfigWidget::descriptionChanged); + connect(QEcalRec::instance(), &QEcalRec::oneFilePerTopicEnabledChangedSignal, this, &ConfigWidget::oneFilePerTopicEnabledChanged); measurementRootDirectoryChanged (QEcalRec::instance()->measRootDir()); measurementNameChanged (QEcalRec::instance()->measName()); @@ -63,6 +65,15 @@ void ConfigWidget::maxFileSizeChanged(unsigned int megabytes) } } +void ConfigWidget::oneFilePerTopicEnabledChanged(bool enabled) +{ + if (enabled != ui_.one_file_per_topic_checkbox->isChecked()) + { + ui_.one_file_per_topic_checkbox->blockSignals(true); + ui_.one_file_per_topic_checkbox->setChecked(enabled); + ui_.one_file_per_topic_checkbox->blockSignals(false); + } +} void ConfigWidget::measurementRootDirectoryChanged(const std::string& root_dir) { if (root_dir.c_str() != ui_.measurement_directory_lineedit->text()) diff --git a/app/rec/rec_gui/src/widgets/config_widget/config_widget.h b/app/rec/rec_gui/src/widgets/config_widget/config_widget.h index c59d12bdc3..b89e4d55bf 100644 --- a/app/rec/rec_gui/src/widgets/config_widget/config_widget.h +++ b/app/rec/rec_gui/src/widgets/config_widget/config_widget.h @@ -35,6 +35,7 @@ private slots: void measurementNameChanged (const std::string& name); void updatePathPreviewAndWarningLabel(); void maxFileSizeChanged (unsigned int megabytes); + void oneFilePerTopicEnabledChanged (bool enabled); void descriptionChanged (const std::string& description); protected: diff --git a/app/rec/rec_gui/src/widgets/config_widget/config_widget.ui b/app/rec/rec_gui/src/widgets/config_widget/config_widget.ui index 3f284c640b..bce95e78fd 100644 --- a/app/rec/rec_gui/src/widgets/config_widget/config_widget.ui +++ b/app/rec/rec_gui/src/widgets/config_widget/config_widget.ui @@ -28,26 +28,57 @@ + + + 10 + 0 + + Meas dir: - + + + + 10 + 0 + + Measurement root directory - - + + + + + 10 + 0 + + - Meas name: + Max file size: - + + + + + 10 + 0 + + + + Description: + + + + @@ -61,6 +92,12 @@ + + + 10 + 0 + + Measurement name (will create a directory inside the measurement root directory) @@ -68,7 +105,7 @@ - + 1 @@ -114,44 +151,27 @@ - - - - Max file size: - - - - - - - Maximum HDF-5 file size (will cause the recorder to split the measurement into multiple files) - - - false - - - MiB - - - 1 - - - 2147483647 - - - 10 + + + + + 10 + 0 + - - - - - Description: + Meas name: - + + + + 10 + 0 + + Description saved to the doc/description.txt file @@ -160,6 +180,60 @@ + + + + + + + 10 + 0 + + + + Maximum HDF-5 file size (will cause the recorder to split the measurement into multiple files) + + + false + + + MiB + + + 1 + + + 2147483647 + + + 10 + + + + + + + + 0 + 0 + + + + + 10 + 0 + + + + Create a separate HDF5 file for each eCAL topic + + + 1 File per topic + + + + + @@ -179,7 +253,6 @@ measurement_directory_lineedit measurement_name_lineedit refresh_path_preview_button - max_file_size_spinbox description_textedit diff --git a/app/rec/rec_server_cli/src/commands/set_config.cpp b/app/rec/rec_server_cli/src/commands/set_config.cpp index ef7b174a10..e114bdc1c8 100644 --- a/app/rec/rec_server_cli/src/commands/set_config.cpp +++ b/app/rec/rec_server_cli/src/commands/set_config.cpp @@ -52,6 +52,7 @@ namespace eCAL &cmdline.meas_name_arg, &cmdline.max_file_size_mib_arg, &cmdline.description_arg, + &cmdline.enable_one_file_per_topic_arg, &cmdline.ftp_server_arg, &cmdline.delete_after_upload_arg, &cmdline.enable_built_in_client_arg, @@ -221,6 +222,14 @@ namespace eCAL if (error) return error; } + + // Enable one file per topic + if (cmdline.enable_one_file_per_topic_arg.isSet()) + { + auto error = setOneFilePerTopicEnabled(rec_server_instance, cmdline.enable_one_file_per_topic_arg.getValue()); + if (error) + return error; + } } return eCAL::rec::Error::ErrorCode::OK; @@ -564,6 +573,12 @@ namespace eCAL rec_server_instance->SetDescription(param); return eCAL::rec::Error::OK; } + + eCAL::rec::Error SetConfig::setOneFilePerTopicEnabled (const std::shared_ptr& rec_server_instance, bool param) + { + rec_server_instance->SetOneFilePerTopicEnabled(param); + return eCAL::rec::Error::OK; + } } } } diff --git a/app/rec/rec_server_cli/src/commands/set_config.h b/app/rec/rec_server_cli/src/commands/set_config.h index 2c9aa34b9a..be1be263f9 100644 --- a/app/rec/rec_server_cli/src/commands/set_config.h +++ b/app/rec/rec_server_cli/src/commands/set_config.h @@ -61,6 +61,7 @@ namespace eCAL , meas_name_arg ("", "meas-name", "Name of the measurement", false, "", "String") , max_file_size_mib_arg ("", "max-file-size", "Maximum file size of the recording files", false, "", "Megabytes") , description_arg ("", "description", "Description stored in the measurement folder.", false, "", "String") + , enable_one_file_per_topic_arg ("", "enable-one-file-per-topic", "Whether to separate each topic in HDF5 file.", false, false, "yes/no") , ftp_server_arg ("", "ftp-server", "The server where to upload to when uploading a measurement. Use \"internal\" for the integrated FTP Server. When using an external FTP Server, provide it in the following form: ftp://USERNAME:PASSWORD@HOSTNAME:PORT/path/to/root_dir", false, "", "FTP_Server") , delete_after_upload_arg ("", "delete-after-upload", "Whether to delete the local measurement files after they have been uploaded to an FTP server.", false, false, "yes/no") , enable_built_in_client_arg("", "enable-built-in-client", "Whether the built-in recorder client of the host-application shall be used for recording. If turned off, the host application will rely on the presence of a separate rec-client for localhost recording.", false, false, "yes/no") @@ -81,6 +82,7 @@ namespace eCAL &description_arg, &ftp_server_arg, &delete_after_upload_arg, + &enable_one_file_per_topic_arg, &enable_built_in_client_arg, &help_arg, &unlabled_arg, @@ -127,6 +129,7 @@ namespace eCAL TCLAP::ValueArg meas_name_arg; TCLAP::ValueArg max_file_size_mib_arg; // => unsigned int! TCLAP::ValueArg description_arg; + CustomTclap::FuzzyValueSwitchArgBool enable_one_file_per_topic_arg; TCLAP::ValueArg ftp_server_arg; CustomTclap::FuzzyValueSwitchArgBool delete_after_upload_arg; CustomTclap::FuzzyValueSwitchArgBool enable_built_in_client_arg; @@ -161,6 +164,7 @@ namespace eCAL static eCAL::rec::Error setMeasName (const std::shared_ptr& rec_server_instance, const std::string& param); static eCAL::rec::Error setMaxFileSize (const std::shared_ptr& rec_server_instance, const std::string& param); static eCAL::rec::Error setDescription (const std::shared_ptr& rec_server_instance, const std::string& param); + static eCAL::rec::Error setOneFilePerTopicEnabled(const std::shared_ptr& rec_server_instance, bool param); }; } } diff --git a/app/rec/rec_server_cli/src/ecal_rec_server_cli.cpp b/app/rec/rec_server_cli/src/ecal_rec_server_cli.cpp index 65bdc9eec3..7607237a87 100644 --- a/app/rec/rec_server_cli/src/ecal_rec_server_cli.cpp +++ b/app/rec/rec_server_cli/src/ecal_rec_server_cli.cpp @@ -195,6 +195,7 @@ int main(int argc, char** argv) TCLAP::ValueArg meas_name_arg ("n", "meas-name", "Name of the measurement, when " + start_recording_arg.toString() + " is set. This will create a folder in the directory provided by " + meas_root_dir_arg.toString() + ". If a configuration file is being loaded, this will override the setting from the config. Not available in remote-control mode.", false, "", "Directory"); TCLAP::ValueArg max_file_size_arg ("", "max-file-size", "Maximum file size of the recording files, when " + start_recording_arg.toString() + " is set. If a configuration file is being loaded, this will override the setting from the config. Not available in remote-control mode.", false, "", "megabytes"); TCLAP::ValueArg description_arg ("", "description", "Description stored in the measurement folder, when " + start_recording_arg.toString() + " is set. If a configuration file is being loaded, this will override the setting from the config. Not available in remote-control mode.", false, "", "String"); + CustomTclap::FuzzyValueSwitchArgBool enable_one_file_per_topic_arg ("", "enable-one-file-per-topic", "Whether to separate each topic in HDF5 file. This helps faster file transfer and less network congestion in case of interest of specific topics only.", false, false, "yes/no"); TCLAP::ValueArg ftp_server_arg ("", "ftp-server", "The server where to upload to when uploading a measurement. Use \"internal\" for the integrated FTP Server. When using an external FTP Server, provide it in the following form: ftp://USERNAME:PASSWORD@HOSTNAME:PORT/path/to/root_dir. If a configuration file is being loaded, this will override the setting from the config. Not available in remote-control mode.", false, "", "FTP_Server"); CustomTclap::FuzzyValueSwitchArgBool delete_after_upload_arg ("", "delete-after-upload", "Whether to delete the local measurement files after they have been uploaded to an FTP server. If a configuration file is being loaded, this will override the setting from the config. Not available in remote-control mode.", false, false, "yes/no"); @@ -238,6 +239,7 @@ int main(int argc, char** argv) &meas_name_arg, &max_file_size_arg, &description_arg, + &enable_one_file_per_topic_arg, &ftp_server_arg, &delete_after_upload_arg, @@ -765,6 +767,39 @@ int main(int argc, char** argv) } } + /************************************************************************/ + /* Enable one file per topic */ + /************************************************************************/ + if (enable_one_file_per_topic_arg.isSet()) + { + bool is_exiting; + { + std::lock_guard ecal_rec_exit_lock(ecal_rec_exit_mutex_); + is_exiting = ctrl_exit_event; + } + + if (!is_exiting) + { + eCAL::rec::Error error(eCAL::rec::Error::ErrorCode::GENERIC_ERROR); + if (rec_server_instance) + { + error = eCAL::rec_cli::command::SetConfig::setOneFilePerTopicEnabled(rec_server_instance, enable_one_file_per_topic_arg.getValue()); + } + else + { + error = eCAL::rec::Error(eCAL::rec::Error::ErrorCode::COMMAND_NOT_AVAILABLE_IN_REMOTE_MODE, enable_one_file_per_topic_arg.toString()); + } + + if (error) + { + std::cerr << "ERROR executing " << enable_one_file_per_topic_arg.toString() << ": " << error.ToString() << std::endl; + set_config_error_occured = true; + } + } + + } + + /************************************************************************/ /* Ftp Server */ /************************************************************************/ diff --git a/app/rec/rec_server_core/CMakeLists.txt b/app/rec/rec_server_core/CMakeLists.txt index aa8cf1864d..c81a7f73ed 100644 --- a/app/rec/rec_server_core/CMakeLists.txt +++ b/app/rec/rec_server_core/CMakeLists.txt @@ -46,8 +46,8 @@ set(source_files src/config/config.h src/config/config_v1.cpp src/config/config_v1.h - src/config/config_v2to3.cpp - src/config/config_v2to3.h + src/config/config_v2to4.cpp + src/config/config_v2to4.h src/recorder/abstract_recorder.h src/recorder/local_recorder.cpp diff --git a/app/rec/rec_server_core/include/rec_server_core/rec_server.h b/app/rec/rec_server_core/include/rec_server_core/rec_server.h index 6259a82993..3c24704058 100644 --- a/app/rec/rec_server_core/include/rec_server_core/rec_server.h +++ b/app/rec/rec_server_core/include/rec_server_core/rec_server.h @@ -119,14 +119,16 @@ namespace eCAL // Job Settings //////////////////////////////////// public: - void SetMeasRootDir (std::string meas_root_dir); - void SetMeasName (std::string meas_name); - void SetMaxFileSizeMib(unsigned int max_file_size_mib); - void SetDescription (std::string description); + void SetMeasRootDir (std::string meas_root_dir); + void SetMeasName (std::string meas_name); + void SetMaxFileSizeMib (unsigned int max_file_size_mib); + void SetOneFilePerTopicEnabled(bool enabled); + void SetDescription (std::string description); std::string GetMeasRootDir () const; std::string GetMeasName () const; int64_t GetMaxFileSizeMib() const; + bool GetOneFilePerTopicEnabled() const; std::string GetDescription () const; //////////////////////////////////// diff --git a/app/rec/rec_server_core/include/rec_server_core/rec_server_config.h b/app/rec/rec_server_core/include/rec_server_core/rec_server_config.h index daefa12e30..f6ba0ff114 100644 --- a/app/rec/rec_server_core/include/rec_server_core/rec_server_config.h +++ b/app/rec/rec_server_core/include/rec_server_core/rec_server_config.h @@ -68,7 +68,8 @@ namespace eCAL RecServerConfig() : root_dir_ ("") , meas_name_ ("") - , max_file_size_ (100) + , max_file_size_ (1000) + , one_file_per_topic_ (false) , description_ ("") , enabled_clients_config_ () , pre_buffer_enabled_ (false) @@ -82,6 +83,7 @@ namespace eCAL std::string root_dir_; std::string meas_name_; int64_t max_file_size_; + bool one_file_per_topic_; std::string description_; std::map enabled_clients_config_; bool pre_buffer_enabled_; diff --git a/app/rec/rec_server_core/src/config/config.cpp b/app/rec/rec_server_core/src/config/config.cpp index a48bd2b008..10e50e6326 100644 --- a/app/rec/rec_server_core/src/config/config.cpp +++ b/app/rec/rec_server_core/src/config/config.cpp @@ -27,7 +27,7 @@ #include #include "config_v1.h" -#include "config_v2to3.h" +#include "config_v2to4.h" namespace eCAL @@ -47,20 +47,21 @@ namespace eCAL } // Set settings that cannot fail - rec_server.SetMeasRootDir (config.root_dir_); - rec_server.SetMeasName (config.meas_name_); - rec_server.SetMaxFileSizeMib (config.max_file_size_); - rec_server.SetDescription (config.description_); - rec_server.SetPreBufferingEnabled(config.pre_buffer_enabled_); - rec_server.SetMaxPreBufferLength (config.pre_buffer_length_); - rec_server.SetUploadConfig (config.upload_config_); + rec_server.SetMeasRootDir (config.root_dir_); + rec_server.SetMeasName (config.meas_name_); + rec_server.SetMaxFileSizeMib (config.max_file_size_); + rec_server.SetDescription (config.description_); + rec_server.SetOneFilePerTopicEnabled (config.one_file_per_topic_); + rec_server.SetPreBufferingEnabled (config.pre_buffer_enabled_); + rec_server.SetMaxPreBufferLength (config.pre_buffer_length_); + rec_server.SetUploadConfig (config.upload_config_); return true; } bool writeConfigToFile(const eCAL::rec_server::RecServerImpl& rec_server, const std::string& path) { - return config_v2to3::writeConfigFile(rec_server, path); + return config_v2to4::writeConfigFile(rec_server, path); } bool readConfigFromFile(eCAL::rec_server::RecServerImpl& rec_server, const std::string& path, int* version) @@ -99,11 +100,11 @@ namespace eCAL for(auto main_config_element = document.FirstChildElement(); main_config_element != nullptr; main_config_element = main_config_element->NextSiblingElement()) { - if (std::string(main_config_element->Name()) == config_v2to3::ELEMENT_NAME_MAIN_CONFIG) + if (std::string(main_config_element->Name()) == config_v2to4::ELEMENT_NAME_MAIN_CONFIG) { - if (main_config_element->Attribute(config_v2to3::ATTRIBUTE_NAME_MAIN_CONFIG_VERSION) != nullptr) + if (main_config_element->Attribute(config_v2to4::ATTRIBUTE_NAME_MAIN_CONFIG_VERSION) != nullptr) { - std::string config_version_number_string = main_config_element->Attribute(config_v2to3::ATTRIBUTE_NAME_MAIN_CONFIG_VERSION); + std::string config_version_number_string = main_config_element->Attribute(config_v2to4::ATTRIBUTE_NAME_MAIN_CONFIG_VERSION); int config_version_number = 0; try { @@ -125,44 +126,28 @@ namespace eCAL } } - if (config_elements.find(NATIVE_CONFIG_VERSION) != config_elements.end()) + // Read a v2 and updwarts config, starting with the latest native version. + for (int v = NATIVE_CONFIG_VERSION; v >= 2; v--) { - // Load a v2 config - bool success = applyConfig(config_v2to3::readConfig(config_elements[NATIVE_CONFIG_VERSION]), rec_server); - if (success && (version != nullptr)) + if (config_elements.find(v) != config_elements.end()) { - (*version) = NATIVE_CONFIG_VERSION; - } - return success; - } - else if (config_elements.find(2) != config_elements.end()) - { - // Load a v2 config - bool success = applyConfig(config_v2to3::readConfig(config_elements[2]), rec_server); - if (success && (version != nullptr)) - { - (*version) = 2; - } - return success; - } - else if (config_elements.find(1) != config_elements.end()) - { - // Load a v1 config - bool success = applyConfig(config_v1::readConfig(config_elements[1]), rec_server); - if (success && (version != nullptr)) - { - (*version) = 1; + // Load config + bool success = applyConfig(config_v2to4::readConfig(config_elements[v]), rec_server); + if (success && (version != nullptr)) + { + (*version) = v; + } + return success; } - return success; } - else + { // Load some other config, although the version is not known for (const auto& version_config_pair : config_elements) { if (version_config_pair.first > NATIVE_CONFIG_VERSION) { - bool success = applyConfig(config_v2to3::readConfig(version_config_pair.second), rec_server); + bool success = applyConfig(config_v2to4::readConfig(version_config_pair.second), rec_server); if (success && (version != nullptr)) { (*version) = version_config_pair.first; @@ -174,7 +159,7 @@ namespace eCAL // Load an un-versioned config if (config_elements.find(0) != config_elements.end()) { - bool success = applyConfig(config_v2to3::readConfig(config_elements[0]), rec_server); + bool success = applyConfig(config_v2to4::readConfig(config_elements[0]), rec_server); if (success && (version != nullptr)) { (*version) = 0; diff --git a/app/rec/rec_server_core/src/config/config.h b/app/rec/rec_server_core/src/config/config.h index 38ca277d0c..a942658a06 100644 --- a/app/rec/rec_server_core/src/config/config.h +++ b/app/rec/rec_server_core/src/config/config.h @@ -27,7 +27,7 @@ namespace eCAL { namespace config { - static constexpr int NATIVE_CONFIG_VERSION = 3; + static constexpr int NATIVE_CONFIG_VERSION = 4; bool writeConfigToFile(const eCAL::rec_server::RecServerImpl& rec_server, const std::string& path); diff --git a/app/rec/rec_server_core/src/config/config_v2to3.cpp b/app/rec/rec_server_core/src/config/config_v2to4.cpp similarity index 95% rename from app/rec/rec_server_core/src/config/config_v2to3.cpp rename to app/rec/rec_server_core/src/config/config_v2to4.cpp index aaded01e04..fee55e79aa 100644 --- a/app/rec/rec_server_core/src/config/config_v2to3.cpp +++ b/app/rec/rec_server_core/src/config/config_v2to4.cpp @@ -17,7 +17,7 @@ * ========================= eCAL LICENSE ================================= */ -#include +#include #include #include @@ -33,7 +33,7 @@ namespace eCAL { namespace rec_server { - namespace config_v2to3 + namespace config_v2to4 { bool writeConfigFile(const eCAL::rec_server::RecServerImpl& rec_server, const std::string& path) { @@ -65,6 +65,12 @@ namespace eCAL max_file_size_element->SetText(std::to_string(rec_server.GetMaxFileSizeMib()).c_str()); main_config_element->InsertEndChild(max_file_size_element); } + { + // one file per topic + auto one_file_per_topic_element = document.NewElement(ELEMENT_NAME_ONE_FILE_PER_TOPIC); + one_file_per_topic_element->SetText(rec_server.GetOneFilePerTopicEnabled() ? "true" : "false"); + main_config_element->InsertEndChild(one_file_per_topic_element); + } { // description auto description_element = document.NewElement(ELEMENT_NAME_DESCRIPTION); @@ -290,6 +296,32 @@ namespace eCAL eCAL::rec::EcalRecLogger::Instance()->warn("Max file size element is missing"); } } + + // one_file_per_topic + { + auto one_file_per_topic_element = main_config_element->FirstChildElement(ELEMENT_NAME_ONE_FILE_PER_TOPIC); + if ((one_file_per_topic_element != nullptr) + && (one_file_per_topic_element->GetText() != nullptr)) + { + std::string one_file_per_topic_string = one_file_per_topic_element->GetText(); + if (one_file_per_topic_string == "true") + { + config_output.one_file_per_topic_ = true; + } + else if (one_file_per_topic_string == "false") + { + config_output.one_file_per_topic_ = false; + } + else + { + eCAL::rec::EcalRecLogger::Instance()->warn("Error parsing one-file-per-topic setting"); + } + } + else + { + eCAL::rec::EcalRecLogger::Instance()->warn("One-file-per-topic element is missing"); + } + } // description { diff --git a/app/rec/rec_server_core/src/config/config_v2to3.h b/app/rec/rec_server_core/src/config/config_v2to4.h similarity index 95% rename from app/rec/rec_server_core/src/config/config_v2to3.h rename to app/rec/rec_server_core/src/config/config_v2to4.h index cb1fef6aa8..f6e8d9a3db 100644 --- a/app/rec/rec_server_core/src/config/config_v2to3.h +++ b/app/rec/rec_server_core/src/config/config_v2to4.h @@ -29,15 +29,16 @@ namespace eCAL { namespace rec_server { - namespace config_v2to3 + namespace config_v2to4 { - constexpr const int NATIVE_CONFIG_VERSION = 3; + constexpr const int NATIVE_CONFIG_VERSION = 4; constexpr const char* ELEMENT_NAME_MAIN_CONFIG = "ecalRecServerConfig"; constexpr const char* ATTRIBUTE_NAME_MAIN_CONFIG_VERSION = "config_version"; constexpr const char* ELEMENT_NAME_ROOT_DIR = "rootDirectory"; constexpr const char* ELEMENT_NAME_MEAS_NAME = "measurementName"; constexpr const char* ELEMENT_NAME_MAX_FILE_SIZE_MIB = "maxFileSizeMib"; + constexpr const char* ELEMENT_NAME_ONE_FILE_PER_TOPIC = "oneFilePerTopic"; // Added in v4 constexpr const char* ELEMENT_NAME_DESCRIPTION = "description"; constexpr const char* ELEMENT_NAME_ENABLED_RECORDERS = "recorders"; constexpr const char* ELEMENT_NAME_ENABLED_RECORDER_ENTRY = "client"; diff --git a/app/rec/rec_server_core/src/proto_helpers.cpp b/app/rec/rec_server_core/src/proto_helpers.cpp index 2cd2c52c2d..9263a42b2c 100644 --- a/app/rec/rec_server_core/src/proto_helpers.cpp +++ b/app/rec/rec_server_core/src/proto_helpers.cpp @@ -90,6 +90,7 @@ namespace eCAL rec_server_config_pb.set_root_dir(rec_server_config.root_dir_); rec_server_config_pb.set_meas_name(rec_server_config.meas_name_); rec_server_config_pb.set_max_file_size_mib(rec_server_config.max_file_size_); + rec_server_config_pb.set_one_file_per_topic(rec_server_config.one_file_per_topic_); rec_server_config_pb.set_description(rec_server_config.description_); rec_server_config_pb.clear_enabled_clients_config(); @@ -323,10 +324,11 @@ namespace eCAL void FromProtobuf(const eCAL::pb::rec_server::RecServerConfig& rec_server_config_pb, eCAL::rec_server::RecServerConfig& rec_server_config) { - rec_server_config.root_dir_ = rec_server_config_pb.root_dir(); - rec_server_config.meas_name_ = rec_server_config_pb.meas_name(); - rec_server_config.max_file_size_ = rec_server_config_pb.max_file_size_mib(); - rec_server_config.description_ = rec_server_config_pb.description(); + rec_server_config.root_dir_ = rec_server_config_pb.root_dir(); + rec_server_config.meas_name_ = rec_server_config_pb.meas_name(); + rec_server_config.max_file_size_ = rec_server_config_pb.max_file_size_mib(); + rec_server_config.one_file_per_topic_ = rec_server_config_pb.one_file_per_topic(); + rec_server_config.description_ = rec_server_config_pb.description(); rec_server_config.enabled_clients_config_.clear(); for (const auto& rec_client_config_pair : rec_server_config_pb.enabled_clients_config()) diff --git a/app/rec/rec_server_core/src/rec_server.cpp b/app/rec/rec_server_core/src/rec_server.cpp index b2fdfb8281..4b35b2722e 100644 --- a/app/rec/rec_server_core/src/rec_server.cpp +++ b/app/rec/rec_server_core/src/rec_server.cpp @@ -90,14 +90,16 @@ namespace eCAL //////////////////////////////////// // Job Settings //////////////////////////////////// - void RecServer::SetMeasRootDir (std::string meas_root_dir) { rec_server_impl_->SetMeasRootDir(meas_root_dir); } - void RecServer::SetMeasName (std::string meas_name) { rec_server_impl_->SetMeasName(meas_name); } - void RecServer::SetMaxFileSizeMib (unsigned int max_file_size_mib) { rec_server_impl_->SetMaxFileSizeMib(max_file_size_mib); } - void RecServer::SetDescription (std::string description) { rec_server_impl_->SetDescription(description); } + void RecServer::SetMeasRootDir (std::string meas_root_dir) { rec_server_impl_->SetMeasRootDir(meas_root_dir); } + void RecServer::SetMeasName (std::string meas_name) { rec_server_impl_->SetMeasName(meas_name); } + void RecServer::SetMaxFileSizeMib (unsigned int max_file_size_mib) { rec_server_impl_->SetMaxFileSizeMib(max_file_size_mib); } + void RecServer::SetOneFilePerTopicEnabled(bool enabled) { rec_server_impl_->SetOneFilePerTopicEnabled(enabled); } + void RecServer::SetDescription (std::string description) { rec_server_impl_->SetDescription(description); } - std::string RecServer::GetMeasRootDir () const { return rec_server_impl_->GetMeasRootDir(); } + std::string RecServer::GetMeasRootDir () const { return rec_server_impl_->GetMeasRootDir(); } std::string RecServer::GetMeasName () const { return rec_server_impl_->GetMeasName(); } int64_t RecServer::GetMaxFileSizeMib() const { return rec_server_impl_->GetMaxFileSizeMib(); } + bool RecServer::GetOneFilePerTopicEnabled() const { return rec_server_impl_->GetOneFilePerTopicEnabled(); } std::string RecServer::GetDescription () const { return rec_server_impl_->GetDescription(); } //////////////////////////////////// diff --git a/app/rec/rec_server_core/src/rec_server_impl.cpp b/app/rec/rec_server_core/src/rec_server_impl.cpp index 312e26e72d..15ba112f5c 100644 --- a/app/rec/rec_server_core/src/rec_server_impl.cpp +++ b/app/rec/rec_server_core/src/rec_server_impl.cpp @@ -950,6 +950,11 @@ namespace eCAL job_config_.SetMaxFileSize(max_file_size_mib); } + void RecServerImpl::SetOneFilePerTopicEnabled(bool enabled) + { + job_config_.SetOneFilePerTopicEnabled(enabled); + } + void RecServerImpl::SetDescription(const std::string& description) { job_config_.SetDescription(description); @@ -971,6 +976,11 @@ namespace eCAL return job_config_.GetMaxFileSize(); } + bool RecServerImpl::GetOneFilePerTopicEnabled() const + { + return job_config_.GetOneFilePerTopicEnabled(); + } + std::string RecServerImpl::GetDescription() const { return job_config_.GetDescription(); @@ -1854,6 +1864,7 @@ namespace eCAL config.root_dir_ = GetMeasRootDir(); config.meas_name_ = GetMeasName(); config.max_file_size_ = GetMaxFileSizeMib(); + config.one_file_per_topic_ = GetOneFilePerTopicEnabled(); config.description_ = GetDescription(); config.enabled_clients_config_ = GetEnabledRecClients(); config.pre_buffer_enabled_ = GetPreBufferingEnabled(); diff --git a/app/rec/rec_server_core/src/rec_server_impl.h b/app/rec/rec_server_core/src/rec_server_impl.h index 20e0222776..7fa8a524bb 100644 --- a/app/rec/rec_server_core/src/rec_server_impl.h +++ b/app/rec/rec_server_core/src/rec_server_impl.h @@ -145,15 +145,17 @@ namespace eCAL // Job Settings // //////////////////////////////////// public: - void SetMeasRootDir (const std::string& meas_root_dir); - void SetMeasName (const std::string& meas_name); - void SetMaxFileSizeMib(int64_t max_file_size_mib); - void SetDescription (const std::string& description); - - std::string GetMeasRootDir () const; - std::string GetMeasName () const; - int64_t GetMaxFileSizeMib() const; - std::string GetDescription () const; + void SetMeasRootDir (const std::string& meas_root_dir); + void SetMeasName (const std::string& meas_name); + void SetMaxFileSizeMib (int64_t max_file_size_mib); + void SetOneFilePerTopicEnabled(bool enabled); + void SetDescription (const std::string& description); + + std::string GetMeasRootDir () const; + std::string GetMeasName () const; + int64_t GetMaxFileSizeMib () const; + bool GetOneFilePerTopicEnabled() const; + std::string GetDescription () const; //////////////////////////////////// // Server Settings // diff --git a/app/rec/rec_server_core/src/recorder/remote_recorder.cpp b/app/rec/rec_server_core/src/recorder/remote_recorder.cpp index 59b9347d55..c8eccde50a 100644 --- a/app/rec/rec_server_core/src/recorder/remote_recorder.cpp +++ b/app/rec/rec_server_core/src/recorder/remote_recorder.cpp @@ -613,11 +613,12 @@ namespace eCAL void RemoteRecorder::SetJobConfig(google::protobuf::Map* job_config_pb, const eCAL::rec::JobConfig& job_config) { - (*job_config_pb)["meas_id"] = std::to_string(job_config.GetJobId()); - (*job_config_pb)["meas_root_dir"] = job_config.GetMeasRootDir(); - (*job_config_pb)["meas_name"] = job_config.GetMeasName(); - (*job_config_pb)["description"] = job_config.GetDescription(); - (*job_config_pb)["max_file_size_mib"] = std::to_string(job_config.GetMaxFileSize()); + (*job_config_pb)["meas_id"] = std::to_string(job_config.GetJobId()); + (*job_config_pb)["meas_root_dir"] = job_config.GetMeasRootDir(); + (*job_config_pb)["meas_name"] = job_config.GetMeasName(); + (*job_config_pb)["description"] = job_config.GetDescription(); + (*job_config_pb)["max_file_size_mib"] = std::to_string(job_config.GetMaxFileSize()); + (*job_config_pb)["one_file_per_topic"] = job_config.GetOneFilePerTopicEnabled() ? "true" : "false"; } void RemoteRecorder::SetUploadConfig(google::protobuf::Map* upload_config_pb, const eCAL::rec::UploadConfig& upload_config) diff --git a/contrib/ecalhdf5/CMakeLists.txt b/contrib/ecalhdf5/CMakeLists.txt index c2a18f03e3..27e850c636 100644 --- a/contrib/ecalhdf5/CMakeLists.txt +++ b/contrib/ecalhdf5/CMakeLists.txt @@ -42,6 +42,8 @@ set(ecalhdf5_src src/eh5_meas_file_v4.h src/eh5_meas_file_v5.cpp src/eh5_meas_file_v5.h + src/eh5_meas_file_writer_v5.cpp + src/eh5_meas_file_writer_v5.h src/eh5_meas_impl.h src/escape.cpp src/escape.h diff --git a/contrib/ecalhdf5/include/ecalhdf5/eh5_meas.h b/contrib/ecalhdf5/include/ecalhdf5/eh5_meas.h index 2889fe6094..d13e08165a 100644 --- a/contrib/ecalhdf5/include/ecalhdf5/eh5_meas.h +++ b/contrib/ecalhdf5/include/ecalhdf5/eh5_meas.h @@ -136,6 +136,28 @@ namespace eCAL **/ void SetMaxSizePerFile(size_t size); + /** + * @brief Whether each Channel shall be writte in its own file + * + * When enabled, data is clustered by channel and each channel is written + * to its own file. The filenames will consist of the basename and the + * channel name. + * + * @return true, if one file per channel is enabled + */ + bool IsOneFilePerChannelEnabled() const; + + /** + * @brief Enable / disable the creation of one individual file per channel + * + * When enabled, data is clustered by channel and each channel is written + * to its own file. The filenames will consist of the basename and the + * channel name. + * + * @param enabled Whether one file shall be created per channel + */ + void SetOneFilePerChannelEnabled(bool enabled); + /** * @brief Get the available channel names of the current opened file / measurement * diff --git a/contrib/ecalhdf5/src/eh5_meas.cpp b/contrib/ecalhdf5/src/eh5_meas.cpp index 4c1eba3774..3375452752 100644 --- a/contrib/ecalhdf5/src/eh5_meas.cpp +++ b/contrib/ecalhdf5/src/eh5_meas.cpp @@ -25,6 +25,8 @@ #include +#include + #include "eh5_meas_dir.h" #include "eh5_meas_file_v1.h" #include "eh5_meas_file_v2.h" @@ -40,8 +42,7 @@ namespace } eCAL::eh5::HDF5Meas::HDF5Meas() -{ -} += default; eCAL::eh5::HDF5Meas::HDF5Meas(const std::string& path, eAccessType access /*= eAccessType::RDONLY*/) { @@ -49,8 +50,7 @@ eCAL::eh5::HDF5Meas::HDF5Meas(const std::string& path, eAccessType access /*= eA } eCAL::eh5::HDF5Meas::~HDF5Meas() -{ -} += default; bool eCAL::eh5::HDF5Meas::Open(const std::string& path, eAccessType access /*= eAccessType::RDONLY*/) { @@ -77,7 +77,7 @@ bool eCAL::eh5::HDF5Meas::Open(const std::string& path, eAccessType access /*= e { hdf_meas_impl_ = std::make_unique(path, access); - if (hdf_meas_impl_->IsOk() == false) + if (!hdf_meas_impl_->IsOk()) { hdf_meas_impl_.reset(); return false; @@ -85,7 +85,7 @@ bool eCAL::eh5::HDF5Meas::Open(const std::string& path, eAccessType access /*= e // no file version at all ? auto file_version = GetFileVersion(); - if (file_version.empty() == true) + if (file_version.empty()) { hdf_meas_impl_.reset(); return false; @@ -190,6 +190,23 @@ void eCAL::eh5::HDF5Meas::SetMaxSizePerFile(size_t size) } } +bool eCAL::eh5::HDF5Meas::IsOneFilePerChannelEnabled() const +{ + if (hdf_meas_impl_ != nullptr) + { + return hdf_meas_impl_->IsOneFilePerChannelEnabled(); + } + return false; +} + +void eCAL::eh5::HDF5Meas::SetOneFilePerChannelEnabled(bool enabled) +{ + if (hdf_meas_impl_ != nullptr) + { + hdf_meas_impl_->SetOneFilePerChannelEnabled(enabled); + } +} + std::set eCAL::eh5::HDF5Meas::GetChannelNames() const { std::set ret_val; @@ -343,7 +360,7 @@ void eCAL::eh5::HDF5Meas::ConnectPreSplitCallback(CallbackFunction cb) { if (hdf_meas_impl_) { - return hdf_meas_impl_->ConnectPreSplitCallback(cb); + return hdf_meas_impl_->ConnectPreSplitCallback(std::move(cb)); } } diff --git a/contrib/ecalhdf5/src/eh5_meas_dir.cpp b/contrib/ecalhdf5/src/eh5_meas_dir.cpp index d644ae2e8a..45ab14eeff 100644 --- a/contrib/ecalhdf5/src/eh5_meas_dir.cpp +++ b/contrib/ecalhdf5/src/eh5_meas_dir.cpp @@ -30,7 +30,6 @@ #include #endif //WIN32 -#include #include #include #include @@ -38,30 +37,26 @@ #include #include -unsigned int kDefaultMaxFileSizeMB = 50; +#include "eh5_meas_file_writer_v5.h" +// TODO: Test the one-file-per-channel setting with gtest +constexpr unsigned int kDefaultMaxFileSizeMB = 1000; eCAL::eh5::HDF5MeasDir::HDF5MeasDir() - : cb_pre_split_(nullptr) - , file_id_(-1) - , file_split_counter_(-1) - , entries_counter_(0) - , max_size_per_file_(kDefaultMaxFileSizeMB * 1024 * 1024) - , access_(RDONLY) -{ - -} + : access_ (RDONLY) // Temporarily set it to RDONLY, so the leading "Close()" from the Open() function will not operate on the uninitialized variable. + , one_file_per_channel_(false) + , max_size_per_file_ (kDefaultMaxFileSizeMB * 1024 * 1024) + , cb_pre_split_ (nullptr) +{} eCAL::eh5::HDF5MeasDir::HDF5MeasDir(const std::string& path, eAccessType access /*= eAccessType::RDONLY*/) - : cb_pre_split_(nullptr) - , file_id_(-1) - , file_split_counter_(-1) - , entries_counter_(0) - , max_size_per_file_(kDefaultMaxFileSizeMB * 1024 * 1024) - , access_(RDONLY) // Temporarily set it to RDONLY, so the leading "Close()" from the Open() function will not operate on the uninitialized variable. + : access_ (access) + , one_file_per_channel_(false) + , max_size_per_file_ (kDefaultMaxFileSizeMB * 1024 * 1024) + , cb_pre_split_ (nullptr) { // call the function via its class becase it's a virtual function that is called in constructor/destructor,- // where the vtable is not created yet or it's destructed. - HDF5MeasDir::Open(path, access); + HDF5MeasDir::Open(path, access_); } eCAL::eh5::HDF5MeasDir::~HDF5MeasDir() @@ -77,6 +72,10 @@ bool eCAL::eh5::HDF5MeasDir::Open(const std::string& path, eAccessType access /* // where the vtable is not created yet or it's destructed. HDF5MeasDir::Close(); + // Check if the given path points to a directory + if (!EcalUtils::Filesystem::IsDir(path, EcalUtils::Filesystem::Current)) + return false; + access_ = access; switch (access) @@ -98,35 +97,20 @@ bool eCAL::eh5::HDF5MeasDir::Close() { if (access_ == eAccessType::CREATE) { - if (!this->IsOk()) return false; - - std::string channels_with_entries; - - for (const auto& channel : channels_) - if (CreateEntriesTableOfContentsFor(channel.first, channel.second.Type, channel.second.Description, channel.second.Entries)) - channels_with_entries += channel.first + ","; - - if ((channels_with_entries.size() > 0) && (channels_with_entries.back() == ',')) - channels_with_entries.pop_back(); - - SetAttribute(file_id_, kChnAttrTitle, channels_with_entries); - - for (auto& channel : channels_) - channel.second.Entries.clear(); - - if (H5Fclose(file_id_) >= 0) - { - file_id_ = -1; - return true; - } - else + // Close all existing file writers + for (auto& file_writer : file_writers_) { - return false; + file_writer.second->Close(); } + + // Clear the list of all file writers, which will delete them + file_writers_.clear(); + + return true; } else { - for (auto file : files_) + for (auto file : file_readers_) { if (file != nullptr) { @@ -136,7 +120,7 @@ bool eCAL::eh5::HDF5MeasDir::Close() } } - files_.clear(); + file_readers_.clear(); channels_info_.clear(); entries_by_id_.clear(); entries_by_chn_.clear(); @@ -151,9 +135,9 @@ bool eCAL::eh5::HDF5MeasDir::IsOk() const { case eCAL::eh5::RDONLY: //case eCAL::eh5::RDWR: - return files_.empty() == false && entries_by_id_.empty() == false; + return !file_readers_.empty() && !entries_by_id_.empty(); case eCAL::eh5::CREATE: - return (file_id_ >= 0); + return true; default: return false; } @@ -162,9 +146,9 @@ bool eCAL::eh5::HDF5MeasDir::IsOk() const std::string eCAL::eh5::HDF5MeasDir::GetFileVersion() const { std::string version; - if (files_.empty() == false) + if (!file_readers_.empty()) { - version = files_.front()->GetFileVersion(); + version = file_readers_.front()->GetFileVersion(); } return version; } @@ -176,7 +160,24 @@ size_t eCAL::eh5::HDF5MeasDir::GetMaxSizePerFile() const void eCAL::eh5::HDF5MeasDir::SetMaxSizePerFile(size_t size) { + // Store to internal variable max_size_per_file_ = size * 1024 * 1024; + + // Update all file writers (if there are any) + for (auto& file_writer : file_writers_) + { + file_writer.second->SetMaxSizePerFile(size); + } +} + +bool eCAL::eh5::HDF5MeasDir::IsOneFilePerChannelEnabled() const +{ + return one_file_per_channel_; +} + +void eCAL::eh5::HDF5MeasDir::SetOneFilePerChannelEnabled(bool enabled) +{ + one_file_per_channel_ = enabled; } std::set eCAL::eh5::HDF5MeasDir::GetChannelNames() const @@ -208,7 +209,9 @@ std::string eCAL::eh5::HDF5MeasDir::GetChannelDescription(const std::string& cha void eCAL::eh5::HDF5MeasDir::SetChannelDescription(const std::string& channel_name, const std::string& description) { - channels_[channel_name].Description = description; + // Get an existing writer or create a new one + auto file_writer_it = GetWriter(channel_name); + file_writer_it->second->SetChannelDescription(channel_name, description); } std::string eCAL::eh5::HDF5MeasDir::GetChannelType(const std::string& channel_name) const @@ -226,7 +229,9 @@ std::string eCAL::eh5::HDF5MeasDir::GetChannelType(const std::string& channel_na void eCAL::eh5::HDF5MeasDir::SetChannelType(const std::string& channel_name, const std::string& type) { - channels_[channel_name].Type = type; + // Get an existing writer or create a new one + auto file_writer_it = GetWriter(channel_name); + file_writer_it->second->SetChannelType(channel_name, type); } long long eCAL::eh5::HDF5MeasDir::GetMinTimestamp(const std::string& channel_name) const @@ -237,7 +242,7 @@ long long eCAL::eh5::HDF5MeasDir::GetMinTimestamp(const std::string& channel_nam if (found != entries_by_chn_.end()) { - if (found->second.empty() == false) + if (!found->second.empty()) { ret_val = found->second.begin()->RcvTimestamp; } @@ -254,7 +259,7 @@ long long eCAL::eh5::HDF5MeasDir::GetMaxTimestamp(const std::string& channel_nam if (found != entries_by_chn_.end()) { - if (found->second.empty() == false) + if (!found->second.empty()) { ret_val = found->second.rbegin()->RcvTimestamp; } @@ -274,7 +279,7 @@ bool eCAL::eh5::HDF5MeasDir::GetEntriesInfo(const std::string& channel_name, Ent entries = found->second; } - return entries.empty() == false; + return !entries.empty(); } bool eCAL::eh5::HDF5MeasDir::GetEntriesInfoRange(const std::string& channel_name, long long begin, long long end, EntryInfoSet& entries) const @@ -324,61 +329,48 @@ bool eCAL::eh5::HDF5MeasDir::GetEntryData(long long entry_id, void* data) const void eCAL::eh5::HDF5MeasDir::SetFileBaseName(const std::string& base_name) { - file_name_ = base_name; + base_name_ = base_name; } bool eCAL::eh5::HDF5MeasDir::AddEntryToFile(const void* data, const unsigned long long& size, const long long& snd_timestamp, const long long& rcv_timestamp, const std::string& channel_name, long long id, long long clock) { - if (!IsOk()) file_id_ = Create(); - if (!IsOk()) - return false; - - hsize_t hsSize = static_cast(size); - - if (EntryFitsTheFile(hsSize) == false) + if ((access_ != CREATE) + || (output_dir_.empty()) + || (base_name_.empty())) { - if (cb_pre_split_ != nullptr) - { - cb_pre_split_(); - } - - if (Create() < 0) - return false; + return false; } - // Create DataSpace with rank 1 and size dimension - auto dataSpace = H5Screate_simple(1, &hsSize, nullptr); - - // Create creation property for dataSpace - auto dsProperty = H5Pcreate(H5P_DATASET_CREATE); - H5Pset_obj_track_times(dsProperty, false); - - // Create dataset in dataSpace - auto dataSet = H5Dcreate(file_id_, std::to_string(entries_counter_).c_str(), H5T_NATIVE_UCHAR, dataSpace, H5P_DEFAULT, dsProperty, H5P_DEFAULT); - - // Write buffer to dataset - herr_t writeStatus = H5Dwrite(dataSet, H5T_NATIVE_UCHAR, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); - - // Close dataset, data space, and data set property - H5Dclose(dataSet); - H5Pclose(dsProperty); - H5Sclose(dataSpace); - - channels_[channel_name].Entries.emplace_back(SEntryInfo(rcv_timestamp, entries_counter_, clock, snd_timestamp, id)); - - entries_counter_++; - - return (writeStatus >= 0); + // Get an existing writer or create a new one + auto file_writer_it = GetWriter(channel_name); + + // Use the writer that was either found or created to actually write the data + return file_writer_it->second->AddEntryToFile(data, size, snd_timestamp, rcv_timestamp, channel_name, id, clock); } void eCAL::eh5::HDF5MeasDir::ConnectPreSplitCallback(CallbackFunction cb) { + // Store the callback function internally cb_pre_split_ = cb; + + // If there are any existing file_writers, add the Callback to all of them. + // They will effectively call the callback, as they are handling the splitting. + for (const auto& file_writer : file_writers_) + { + file_writer.second->ConnectPreSplitCallback(cb_pre_split_); + } } void eCAL::eh5::HDF5MeasDir::DisconnectPreSplitCallback() { + // Clear internal copy of the callback cb_pre_split_ = nullptr; + + // If there are any existing file_writers, remove the Callback from all of them. + for (const auto& file_writer : file_writers_) + { + file_writer.second->DisconnectPreSplitCallback(); + } } std::list eCAL::eh5::HDF5MeasDir::GetHdfFiles(const std::string& path) const @@ -433,7 +425,7 @@ std::list eCAL::eh5::HDF5MeasDir::GetHdfFiles(const std::string& pa } else { - if (HasHdf5Extension(d_name) == true) + if (HasHdf5Extension(d_name)) paths.push_back(path + "/" + d_name); } } @@ -445,43 +437,6 @@ std::list eCAL::eh5::HDF5MeasDir::GetHdfFiles(const std::string& pa return paths; } -hid_t eCAL::eh5::HDF5MeasDir::Create() -{ - if (output_dir_.empty()) return -1; - - if (!EcalUtils::Filesystem::IsDir(output_dir_, EcalUtils::Filesystem::OsStyle::Current) - && !EcalUtils::Filesystem::MkPath(output_dir_, EcalUtils::Filesystem::OsStyle::Current)) - return -1; - - if (file_name_.empty()) return -1; - - if (IsOk() && !Close()) return -1; - - file_split_counter_++; - - std::string filePath = output_dir_ + "/" + file_name_; - - if (file_split_counter_ > 0) - filePath += "_" + std::to_string(file_split_counter_); - - filePath += ".hdf5"; - - // create file access property - hid_t fileAccessPropery = H5Pcreate(H5P_FILE_ACCESS); - // create file create property - hid_t fileCreateProperty = H5Pcreate(H5P_FILE_CREATE); - - // Create hdf file and get file id - file_id_ = H5Fcreate(filePath.c_str(), H5F_ACC_TRUNC, fileCreateProperty, fileAccessPropery); - - if (file_id_ >= 0) - SetAttribute(file_id_, kFileVerAttrTitle, "5.0"); - else - file_split_counter_--; - - return file_id_; -} - bool eCAL::eh5::HDF5MeasDir::OpenRX(const std::string& path, eAccessType access /*= eAccessType::RDONLY*/) { if (access != eAccessType::RDONLY /*&& access != eAccessType::RDWR*/) return false; @@ -494,7 +449,7 @@ bool eCAL::eh5::HDF5MeasDir::OpenRX(const std::string& path, eAccessType access { auto reader = new eCAL::eh5::HDF5Meas(file_path); - if (reader->IsOk() == true) + if (reader->IsOk()) { auto channels = reader->GetChannelNames(); for (const auto& channel : channels) @@ -508,7 +463,7 @@ bool eCAL::eh5::HDF5MeasDir::OpenRX(const std::string& path, eAccessType access } else { - if (description.empty() == false) + if (!description.empty()) { channels_info_[escaped_name].description = description; } @@ -517,7 +472,7 @@ bool eCAL::eh5::HDF5MeasDir::OpenRX(const std::string& path, eAccessType access channels_info_[escaped_name].files.push_back(reader); EntryInfoSet entries; - if (reader->GetEntriesInfo(channel, entries) == true) + if (reader->GetEntriesInfo(channel, entries)) { for (auto entry : entries) { @@ -528,7 +483,7 @@ bool eCAL::eh5::HDF5MeasDir::OpenRX(const std::string& path, eAccessType access } } } - files_.push_back(reader); + file_readers_.push_back(reader); } else { @@ -537,95 +492,32 @@ bool eCAL::eh5::HDF5MeasDir::OpenRX(const std::string& path, eAccessType access reader = nullptr; } } - return files_.empty() == false; + return !file_readers_.empty(); } -bool eCAL::eh5::HDF5MeasDir::SetAttribute(const hid_t& id, const std::string& name, const std::string& value) +::eCAL::eh5::HDF5MeasDir::FileWriterMap::iterator eCAL::eh5::HDF5MeasDir::GetWriter(const std::string& channel_name) { - if (id < 0) return false; - - if (H5Aexists(id, name.c_str()) > 0) - H5Adelete(id, name.c_str()); - // create scalar dataset - hid_t scalarDataset = H5Screate(H5S_SCALAR); - - // create new string data type - hid_t stringDataType = H5Tcopy(H5T_C_S1); - - // if attribute's value length exists, allocate space for it - if (value.length() > 0) - H5Tset_size(stringDataType, value.length()); - - // create attribute - hid_t attribute = H5Acreate(id, name.c_str(), stringDataType, scalarDataset, H5P_DEFAULT, H5P_DEFAULT); - - if (attribute < 0) return false; - - // write attribute value to attribute - herr_t writeStatus = H5Awrite(attribute, stringDataType, value.c_str()); - if (writeStatus < 0) return false; - - // close attribute - H5Aclose(attribute); - // close scalar dataset - H5Sclose(scalarDataset); - // close string data type - H5Tclose(stringDataType); - - return true; -} - -bool eCAL::eh5::HDF5MeasDir::EntryFitsTheFile(const hsize_t& size) const -{ - hsize_t fileSize = 0; - herr_t status = GetFileSize(fileSize); - - // check if buffer fits the current file - return (status && ((fileSize + size) <= max_size_per_file_)); -} - -bool eCAL::eh5::HDF5MeasDir::GetFileSize(hsize_t& size) const -{ - size = 0; - - if (!IsOk()) return false; - - return H5Fget_filesize(file_id_, &size) >= 0; -} - -bool eCAL::eh5::HDF5MeasDir::CreateEntriesTableOfContentsFor(const std::string& channelName, const std::string& channelType, const std::string& channelDescription, const EntryInfoVect& entries) -{ - if (!IsOk()) return false; - - const size_t dataSetsSize = entries.size(); - - if (dataSetsSize == 0) return false; - - hsize_t dims[2] = { dataSetsSize, 5 }; - - // Create DataSpace with rank 2 and size dimension - auto dataSpace = H5Screate_simple(2, dims, nullptr); - - // Create creation property for dataSpace - auto dsProperty = H5Pcreate(H5P_DATASET_CREATE); - H5Pset_obj_track_times(dsProperty, false); - - auto dataSet = H5Dcreate(file_id_, channelName.c_str(), H5T_NATIVE_LLONG, dataSpace, H5P_DEFAULT, dsProperty, H5P_DEFAULT); - - if (dataSet < 0) return false; - - SetAttribute(dataSet, kChnTypeAttrTitle, channelType); - SetAttribute(dataSet, kChnDescAttrTitle, channelDescription); - - // Write buffer to dataset - herr_t writeStatus = H5Dwrite(dataSet, H5T_NATIVE_LLONG, H5S_ALL, H5S_ALL, H5P_DEFAULT, entries.data()); - if (writeStatus < 0) return false; - - // Close dataset, data space, and data set property - H5Dclose(dataSet); - H5Pclose(dsProperty); - H5Sclose(dataSpace); + // Look for an existing writer. When creating 1 file per channel, the channel + // name is used as key. Otherwise, emptystring is used as "generic" key and + // the same writer is used for all channels. + FileWriterMap::iterator file_writer_it = file_writers_.find(one_file_per_channel_ ? channel_name : ""); + if (file_writer_it == file_writers_.end()) + { + // No appropriate file writer was found. Let's create a new one! + file_writer_it = file_writers_.emplace(one_file_per_channel_ ? channel_name : "", std::make_unique<::eCAL::eh5::HDF5MeasFileWriterV5>()).first; + + // Set the current parameters to the new file writer + file_writer_it->second->SetMaxSizePerFile(max_size_per_file_); + file_writer_it->second->SetOneFilePerChannelEnabled(one_file_per_channel_); + file_writer_it->second->SetFileBaseName(one_file_per_channel_ ? (base_name_ + "_" + channel_name) : (base_name_)); + if (cb_pre_split_) + file_writer_it->second->ConnectPreSplitCallback(cb_pre_split_); + + // Open the writer + file_writer_it->second->Open(output_dir_); + } - return true; + // The iterator is either what we found or what we created. In either way it + // will be valid and can be returned. + return file_writer_it; } - diff --git a/contrib/ecalhdf5/src/eh5_meas_dir.h b/contrib/ecalhdf5/src/eh5_meas_dir.h index 10a3d3a282..23791fe83a 100644 --- a/contrib/ecalhdf5/src/eh5_meas_dir.h +++ b/contrib/ecalhdf5/src/eh5_meas_dir.h @@ -27,6 +27,7 @@ #include #include #include +#include #include "eh5_meas_impl.h" @@ -56,7 +57,7 @@ namespace eCAL /** * @brief Destructor **/ - ~HDF5MeasDir(); + ~HDF5MeasDir() override; /** * @brief Open file @@ -103,6 +104,28 @@ namespace eCAL **/ void SetMaxSizePerFile(size_t size) override; + /** + * @brief Whether each Channel shall be writte in its own file + * + * When enabled, data is clustered by channel and each channel is written + * to its own file. The filenames will consist of the basename and the + * channel name. + * + * @return true, if one file per channel is enabled + */ + bool IsOneFilePerChannelEnabled() const override; + + /** + * @brief Enable / disable the creation of one individual file per channel + * + * When enabled, data is clustered by channel and each channel is written + * to its own file. The filenames will consist of the basename and the + * channel name. + * + * @param enabled Whether one file shall be created per channel + */ + void SetOneFilePerChannelEnabled(bool enabled) override; + /** * @brief Get the available channel names of the current opened file / measurement * @@ -250,6 +273,10 @@ namespace eCAL **/ void DisconnectPreSplitCallback() override; + + // ===================================================================== + // ==== Reading Files + // ===================================================================== protected: struct ChannelInfo { @@ -257,7 +284,7 @@ namespace eCAL std::string description; std::list files; - ChannelInfo() {} + ChannelInfo() = default; ChannelInfo(const std::string& type_, const std::string& description_) : type(type_) , description(description_) @@ -266,7 +293,7 @@ namespace eCAL struct EntryInfo { - long long file_id; + long long file_id; const eCAL::eh5::HDF5Meas* reader; EntryInfo() : file_id(0), reader(nullptr) {} @@ -282,7 +309,7 @@ namespace eCAL typedef std::unordered_map EntriesByIdUMap; typedef std::unordered_map EntriesByChannelUMap; - HDF5Files files_; + HDF5Files file_readers_; ChannelInfoUMap channels_info_; EntriesByIdUMap entries_by_id_; EntriesByChannelUMap entries_by_chn_; @@ -296,14 +323,7 @@ namespace eCAL typedef std::map Channels; - std::string output_dir_; - std::string file_name_; Channels channels_; - CallbackFunction cb_pre_split_; - hid_t file_id_; - int file_split_counter_; - unsigned long long entries_counter_; - size_t max_size_per_file_; eAccessType access_; std::list GetHdfFiles(const std::string& path) const; @@ -315,57 +335,39 @@ namespace eCAL return std::equal(end.rbegin(), end.rend(), str.rbegin()); } - /** - * @brief Creates the actual file - * - * @return file ID, file was not created if id is negative - **/ - hid_t Create(); - bool OpenRX(const std::string& path, eAccessType access /*= eAccessType::RDONLY*/); - /** - * @brief Set attribute to object(file, entry...) - * - * @param id ID of the attributes parent - * @param name Name of the attribute - * @param value Value of the attribute - * - * @return true if succeeds, false if it fails - **/ - bool SetAttribute(const hid_t& id, const std::string& name, const std::string& value); - /** - * @brief Checks if current file size + entry size does not exceed the maximum allowed size of the file - * - * @param size Size of the entry in bytes - * - * @return true if entry can be saved in current file, false if it can not be added to the current file - **/ - bool EntryFitsTheFile(const hsize_t& size) const; + // ===================================================================== + // ==== Writing files + // ===================================================================== + protected: + typedef std::unordered_map> FileWriterMap; - /** - * @brief Gets the size of the file - * - * @param size Size of the file in bytes - * - * @return true if succeeds, false if it fails - **/ - bool GetFileSize(hsize_t& size) const; + std::string output_dir_; //!< The directory where the HDF5 files shall be placed when in CREATE mode + std::string base_name_; //!< The filename of HDF5 files when in CREATE mode. Will be postfixed by the channel name when in one_file_per_channel_ mode. Will be further postfixed by a number when the files are splitted. + bool one_file_per_channel_; //!< If true, one FileWriter will be created for each channel. + FileWriterMap file_writers_; //!< Map of {ChannelName -> FileWriter}. Grows for each new channel, if one_file_per_channel_ is true. Contains only one "" key otherwise that is used for all channels. - /** - * @brief Creates the entries "table of contents" (timestamp + entry id) - * (Call it just before closing the file) - * - * @param channelName name for the dataset - * @param channelType type for the dataset - * @param channelDescription description for the dataset - * @param entries entries for given channel - * - * @return true if succeeds, false if it fails - **/ - bool CreateEntriesTableOfContentsFor(const std::string& channelName, const std::string& channelType, const std::string& channelDescription, const EntryInfoVect& entries); + size_t max_size_per_file_; //!< Maximum file size after which the File Writer shall split + CallbackFunction cb_pre_split_; //!< Callback that is executed before a new hdf5 file is created during splitting. Will be executed by each file writer individually. + protected: + /** + * @brief Returns a writer for the given channel name + * + * If one_file_per_channel_ is true, one writer for each channel will be + * create and / or returned. Otherwise, only one writer will exist and + * be reused for all channels. + * + * New writers will be initialized with the split size and callback that + * were set in this class. + * + * @param channel_name The channel name to return a writer for + * + * @return an iterator to the writer + */ + FileWriterMap::iterator GetWriter(const std::string& channel_name); }; } // namespace eh5 } // namespace eCAL diff --git a/contrib/ecalhdf5/src/eh5_meas_file_v1.cpp b/contrib/ecalhdf5/src/eh5_meas_file_v1.cpp index 0da8130d43..02fed6dbec 100644 --- a/contrib/ecalhdf5/src/eh5_meas_file_v1.cpp +++ b/contrib/ecalhdf5/src/eh5_meas_file_v1.cpp @@ -76,7 +76,7 @@ bool eCAL::eh5::HDF5MeasFileV1::Open(const std::string& path, eAccessType access file_id_ = H5Fopen(path.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT); - if (HDF5MeasFileV1::IsOk() == true) + if (HDF5MeasFileV1::IsOk()) { auto channels = HDF5MeasFileV1::GetChannelNames(); if (channels.size() == 1) @@ -94,7 +94,7 @@ bool eCAL::eh5::HDF5MeasFileV1::Open(const std::string& path, eAccessType access bool eCAL::eh5::HDF5MeasFileV1::Close() { - if (HDF5MeasFileV1::IsOk() == true && H5Fclose(file_id_) >= 0) + if (HDF5MeasFileV1::IsOk() && H5Fclose(file_id_) >= 0) { file_id_ = -1; return true; @@ -132,6 +132,17 @@ void eCAL::eh5::HDF5MeasFileV1::SetMaxSizePerFile(size_t /*size*/) ReportUnsupportedAction(); } +bool eCAL::eh5::HDF5MeasFileV1::IsOneFilePerChannelEnabled() const +{ + ReportUnsupportedAction(); + return false; +} + +void eCAL::eh5::HDF5MeasFileV1::SetOneFilePerChannelEnabled(bool /*enabled*/) +{ + ReportUnsupportedAction(); +} + std::set eCAL::eh5::HDF5MeasFileV1::GetChannelNames() const { std::set channels; @@ -139,7 +150,7 @@ std::set eCAL::eh5::HDF5MeasFileV1::GetChannelNames() const std::string channel_name; GetAttributeValue(file_id_, kChnNameAttribTitle, channel_name); - if (channel_name.empty() == false) + if (!channel_name.empty()) channels.insert(channel_name); return channels; @@ -157,7 +168,7 @@ std::string eCAL::eh5::HDF5MeasFileV1::GetChannelDescription(const std::string& { std::string description; - if (EcalUtils::String::Icompare(channel_name, channel_name_) == true) + if (EcalUtils::String::Icompare(channel_name, channel_name_)) GetAttributeValue(file_id_, kChnDescAttrTitle, description); return description; @@ -172,7 +183,7 @@ std::string eCAL::eh5::HDF5MeasFileV1::GetChannelType(const std::string& channel { std::string type; - if (EcalUtils::String::Icompare(channel_name, channel_name_) == true) + if (EcalUtils::String::Icompare(channel_name, channel_name_)) GetAttributeValue(file_id_, kChnTypeAttrTitle, type); return type; @@ -187,7 +198,7 @@ long long eCAL::eh5::HDF5MeasFileV1::GetMinTimestamp(const std::string& /*channe { long long ret_val = 0; - if (entries_.empty() == false) + if (!entries_.empty()) { ret_val = entries_.begin()->RcvTimestamp; } @@ -199,7 +210,7 @@ long long eCAL::eh5::HDF5MeasFileV1::GetMaxTimestamp(const std::string& /*channe { long long ret_val = 0; - if (entries_.empty() == false) + if (!entries_.empty()) { ret_val = entries_.rbegin()->RcvTimestamp; } @@ -211,9 +222,9 @@ bool eCAL::eh5::HDF5MeasFileV1::GetEntriesInfo(const std::string& channel_name, { entries.clear(); - if (EcalUtils::String::Icompare(channel_name, channel_name_) == false) return false; + if (!EcalUtils::String::Icompare(channel_name, channel_name_)) return false; - if (HDF5MeasFileV1::IsOk() == false) return false; + if (!HDF5MeasFileV1::IsOk()) return false; auto dataset_id = H5Dopen(file_id_, kTimestampAttrTitle.c_str(), H5P_DEFAULT); @@ -247,7 +258,7 @@ bool eCAL::eh5::HDF5MeasFileV1::GetEntriesInfoRange(const std::string& /*channel { bool ret_val = false; - if (entries_.empty() == false) + if (!entries_.empty()) { if (begin == 0) begin = entries.begin()->RcvTimestamp; if (end == 0) end = entries.rbegin()->RcvTimestamp; @@ -281,7 +292,7 @@ bool eCAL::eh5::HDF5MeasFileV1::GetEntryData(long long entry_id, void* data) con { if (data == nullptr) return false; - if (this->IsOk() == false) return false; + if (!this->IsOk()) return false; auto dataset_id = H5Dopen(file_id_, std::to_string(entry_id).c_str(), H5P_DEFAULT); @@ -320,7 +331,7 @@ void eCAL::eh5::HDF5MeasFileV1::DisconnectPreSplitCallback() ReportUnsupportedAction(); } -bool eCAL::eh5::HDF5MeasFileV1::GetAttributeValue(hid_t obj_id, const std::string& name, std::string& value) const +bool eCAL::eh5::HDF5MeasFileV1::GetAttributeValue(hid_t obj_id, const std::string& name, std::string& value) { bool ret_val = false; // empty attribute value @@ -328,7 +339,7 @@ bool eCAL::eh5::HDF5MeasFileV1::GetAttributeValue(hid_t obj_id, const std::strin if (obj_id < 0) return false; // check if attribute exists - if (H5Aexists(obj_id, name.c_str())) + if (H5Aexists(obj_id, name.c_str()) != 0) { // open attribute by name, getting the attribute index hid_t attr_id = H5Aopen_name(obj_id, name.c_str()); @@ -374,7 +385,7 @@ bool eCAL::eh5::HDF5MeasFileV1::GetAttributeValue(hid_t obj_id, const std::strin return ret_val; } -void eCAL::eh5::HDF5MeasFileV1::ReportUnsupportedAction() const +void eCAL::eh5::HDF5MeasFileV1::ReportUnsupportedAction() { std::cout << "eCALHDF5 file version bellow 2.0 support only readonly access type. Desired action not supported.\n"; } diff --git a/contrib/ecalhdf5/src/eh5_meas_file_v1.h b/contrib/ecalhdf5/src/eh5_meas_file_v1.h index 4615809310..38cf339eb7 100644 --- a/contrib/ecalhdf5/src/eh5_meas_file_v1.h +++ b/contrib/ecalhdf5/src/eh5_meas_file_v1.h @@ -48,7 +48,7 @@ namespace eCAL /** * @brief Destructor **/ - ~HDF5MeasFileV1(); + ~HDF5MeasFileV1() override; /** * @brief Open file @@ -95,6 +95,29 @@ namespace eCAL **/ void SetMaxSizePerFile(size_t size) override; + /** + * @brief Whether each Channel shall be writte in its own file + * + * When enabled, data is clustered by channel and each channel is written + * to its own file. The filenames will consist of the basename and the + * channel name. + * + * @return true, if one file per channel is enabled + */ + bool IsOneFilePerChannelEnabled() const override; + + /** + * @brief Enable / disable the creation of one individual file per channel + * + * When enabled, data is clustered by channel and each channel is written + * to its own file. The filenames will consist of the basename and the + * channel name. + * + * @param enabled Whether one file shall be created per channel + */ + void SetOneFilePerChannelEnabled(bool enabled) override; + + /** * @brief Get the available channel names of the current opened file / measurement * @@ -256,12 +279,12 @@ namespace eCAL * * @return true if succeeds, false if it fails **/ - bool GetAttributeValue(hid_t obj_id, const std::string& name, std::string& value) const; + static bool GetAttributeValue(hid_t obj_id, const std::string& name, std::string& value) ; /** * @brief Reports the use of an unsupported interface method. **/ - void ReportUnsupportedAction() const; + static void ReportUnsupportedAction() ; }; } // namespace eh5 diff --git a/contrib/ecalhdf5/src/eh5_meas_file_v2.cpp b/contrib/ecalhdf5/src/eh5_meas_file_v2.cpp index 250d65aa85..2033b777c0 100644 --- a/contrib/ecalhdf5/src/eh5_meas_file_v2.cpp +++ b/contrib/ecalhdf5/src/eh5_meas_file_v2.cpp @@ -74,7 +74,7 @@ bool eCAL::eh5::HDF5MeasFileV2::Open(const std::string& path, eAccessType access bool eCAL::eh5::HDF5MeasFileV2::Close() { - if (HDF5MeasFileV2::IsOk() == true && H5Fclose(file_id_) >= 0) + if (HDF5MeasFileV2::IsOk() && H5Fclose(file_id_) >= 0) { file_id_ = -1; return true; @@ -110,6 +110,16 @@ void eCAL::eh5::HDF5MeasFileV2::SetMaxSizePerFile(size_t /*size*/) { } + +bool eCAL::eh5::HDF5MeasFileV2::IsOneFilePerChannelEnabled() const +{ + return false; +} + +void eCAL::eh5::HDF5MeasFileV2::SetOneFilePerChannelEnabled(bool /*enabled*/) +{ +} + std::set eCAL::eh5::HDF5MeasFileV2::GetChannelNames() const { std::set channels_set; @@ -138,7 +148,7 @@ std::string eCAL::eh5::HDF5MeasFileV2::GetChannelDescription(const std::string& { std::string description; - if (this->IsOk() == true) + if (this->IsOk()) { auto dataset_id = H5Dopen(file_id_, channel_name.c_str(), H5P_DEFAULT); if (dataset_id >= 0) @@ -159,7 +169,7 @@ std::string eCAL::eh5::HDF5MeasFileV2::GetChannelType(const std::string& channel { std::string type; - if (this->IsOk() == true) + if (this->IsOk()) { auto dataset_id = H5Dopen(file_id_, channel_name.c_str(), H5P_DEFAULT); if (dataset_id >= 0) @@ -181,7 +191,7 @@ long long eCAL::eh5::HDF5MeasFileV2::GetMinTimestamp(const std::string& channel_ long long ret_val = 0; EntryInfoSet entries; - if (GetEntriesInfo(channel_name, entries) == true && entries.empty() == false) + if (GetEntriesInfo(channel_name, entries) && !entries.empty()) { ret_val = entries.begin()->RcvTimestamp; } @@ -194,7 +204,7 @@ long long eCAL::eh5::HDF5MeasFileV2::GetMaxTimestamp(const std::string& channel_ long long ret_val = 0; EntryInfoSet entries; - if (GetEntriesInfo(channel_name, entries) == true && entries.empty() == false) + if (GetEntriesInfo(channel_name, entries) && !entries.empty()) { ret_val = entries.rbegin()->RcvTimestamp; } @@ -242,7 +252,7 @@ bool eCAL::eh5::HDF5MeasFileV2::GetEntriesInfoRange(const std::string& channel_n EntryInfoSet all_entries; entries.clear(); - if (GetEntriesInfo(channel_name, all_entries) == true && all_entries.empty() == false) + if (GetEntriesInfo(channel_name, all_entries) && !all_entries.empty()) { if (begin == 0) begin = entries.begin()->RcvTimestamp; if (end == 0) end = entries.rbegin()->RcvTimestamp; @@ -276,7 +286,7 @@ bool eCAL::eh5::HDF5MeasFileV2::GetEntryData(long long entry_id, void* data) con { if (data == nullptr) return false; - if (this->IsOk() == false) return false; + if (!this->IsOk()) return false; auto dataset_id = H5Dopen(file_id_, std::to_string(entry_id).c_str(), H5P_DEFAULT); @@ -312,7 +322,7 @@ void eCAL::eh5::HDF5MeasFileV2::DisconnectPreSplitCallback() { } -bool eCAL::eh5::HDF5MeasFileV2::GetAttributeValue(hid_t obj_id, const std::string& name, std::string& value) const +bool eCAL::eh5::HDF5MeasFileV2::GetAttributeValue(hid_t obj_id, const std::string& name, std::string& value) { bool ret_val = false; // empty attribute value @@ -320,7 +330,7 @@ bool eCAL::eh5::HDF5MeasFileV2::GetAttributeValue(hid_t obj_id, const std::strin if (obj_id < 0) return false; // check if attribute exists - if (H5Aexists(obj_id, name.c_str())) + if (H5Aexists(obj_id, name.c_str()) != 0) { // open attribute by name, getting the attribute index hid_t attr_id = H5Aopen_name(obj_id, name.c_str()); diff --git a/contrib/ecalhdf5/src/eh5_meas_file_v2.h b/contrib/ecalhdf5/src/eh5_meas_file_v2.h index 6d251acc7c..6385f4f9fc 100644 --- a/contrib/ecalhdf5/src/eh5_meas_file_v2.h +++ b/contrib/ecalhdf5/src/eh5_meas_file_v2.h @@ -48,7 +48,7 @@ namespace eCAL /** * @brief Destructor **/ - ~HDF5MeasFileV2(); + ~HDF5MeasFileV2() override; /** * @brief Open file @@ -95,6 +95,29 @@ namespace eCAL **/ void SetMaxSizePerFile(size_t size) override; + /** + * @brief Whether each Channel shall be writte in its own file + * + * When enabled, data is clustered by channel and each channel is written + * to its own file. The filenames will consist of the basename and the + * channel name. + * + * @return true, if one file per channel is enabled + */ + bool IsOneFilePerChannelEnabled() const override; + + /** + * @brief Enable / disable the creation of one individual file per channel + * + * When enabled, data is clustered by channel and each channel is written + * to its own file. The filenames will consist of the basename and the + * channel name. + * + * @param enabled Whether one file shall be created per channel + */ + void SetOneFilePerChannelEnabled(bool enabled) override; + + /** * @brief Get the available channel names of the current opened file / measurement * @@ -254,7 +277,7 @@ namespace eCAL * * @return true if succeeds, false if it fails **/ - bool GetAttributeValue(hid_t obj_id, const std::string& name, std::string& value) const; + static bool GetAttributeValue(hid_t obj_id, const std::string& name, std::string& value) ; }; } // namespace eh5 diff --git a/contrib/ecalhdf5/src/eh5_meas_file_v3.cpp b/contrib/ecalhdf5/src/eh5_meas_file_v3.cpp index 88b0167293..0469d80619 100644 --- a/contrib/ecalhdf5/src/eh5_meas_file_v3.cpp +++ b/contrib/ecalhdf5/src/eh5_meas_file_v3.cpp @@ -36,12 +36,10 @@ namespace eCAL } HDF5MeasFileV3::HDF5MeasFileV3() - { - } + = default; HDF5MeasFileV3::~HDF5MeasFileV3() - { - } + = default; bool HDF5MeasFileV3::GetEntriesInfo(const std::string& channel_name, EntryInfoSet& entries) const { diff --git a/contrib/ecalhdf5/src/eh5_meas_file_v3.h b/contrib/ecalhdf5/src/eh5_meas_file_v3.h index 396a274802..716bf00a7c 100644 --- a/contrib/ecalhdf5/src/eh5_meas_file_v3.h +++ b/contrib/ecalhdf5/src/eh5_meas_file_v3.h @@ -49,7 +49,7 @@ namespace eCAL /** * @brief Destructor **/ - ~HDF5MeasFileV3(); + ~HDF5MeasFileV3() override; /** * @brief Gets the header info for all data entries for the given channel @@ -60,7 +60,7 @@ namespace eCAL * * @return true if succeeds, false if it fails **/ - bool GetEntriesInfo(const std::string& channel_name, EntryInfoSet& entries) const; + bool GetEntriesInfo(const std::string& channel_name, EntryInfoSet& entries) const override; }; } // namespace eh5 } // namespace eCAL diff --git a/contrib/ecalhdf5/src/eh5_meas_file_v4.cpp b/contrib/ecalhdf5/src/eh5_meas_file_v4.cpp index e068be0220..ae5d125d76 100644 --- a/contrib/ecalhdf5/src/eh5_meas_file_v4.cpp +++ b/contrib/ecalhdf5/src/eh5_meas_file_v4.cpp @@ -36,12 +36,10 @@ namespace eCAL } HDF5MeasFileV4::HDF5MeasFileV4() - { - } + = default; HDF5MeasFileV4::~HDF5MeasFileV4() - { - } + = default; bool HDF5MeasFileV4::GetEntriesInfo(const std::string& channel_name, EntryInfoSet& entries) const { diff --git a/contrib/ecalhdf5/src/eh5_meas_file_v4.h b/contrib/ecalhdf5/src/eh5_meas_file_v4.h index 259e159dbd..f9d1139752 100644 --- a/contrib/ecalhdf5/src/eh5_meas_file_v4.h +++ b/contrib/ecalhdf5/src/eh5_meas_file_v4.h @@ -49,7 +49,7 @@ namespace eCAL /** * @brief Destructor **/ - ~HDF5MeasFileV4(); + ~HDF5MeasFileV4() override; /** * @brief Gets the header info for all data entries for the given channel @@ -60,7 +60,7 @@ namespace eCAL * * @return true if succeeds, false if it fails **/ - bool GetEntriesInfo(const std::string& channel_name, EntryInfoSet& entries) const; + bool GetEntriesInfo(const std::string& channel_name, EntryInfoSet& entries) const override; }; } // namespace eh5 } // namespace eCAL diff --git a/contrib/ecalhdf5/src/eh5_meas_file_v5.cpp b/contrib/ecalhdf5/src/eh5_meas_file_v5.cpp index 2d715e6178..39960e6015 100644 --- a/contrib/ecalhdf5/src/eh5_meas_file_v5.cpp +++ b/contrib/ecalhdf5/src/eh5_meas_file_v5.cpp @@ -36,12 +36,10 @@ namespace eCAL } HDF5MeasFileV5::HDF5MeasFileV5() - { - } + = default; HDF5MeasFileV5::~HDF5MeasFileV5() - { - } + = default; bool HDF5MeasFileV5::GetEntriesInfo(const std::string& channel_name, EntryInfoSet& entries) const { diff --git a/contrib/ecalhdf5/src/eh5_meas_file_v5.h b/contrib/ecalhdf5/src/eh5_meas_file_v5.h index d057939009..9188616b15 100644 --- a/contrib/ecalhdf5/src/eh5_meas_file_v5.h +++ b/contrib/ecalhdf5/src/eh5_meas_file_v5.h @@ -49,7 +49,7 @@ namespace eCAL /** * @brief Destructor **/ - ~HDF5MeasFileV5(); + ~HDF5MeasFileV5() override; /** * @brief Gets the header info for all data entries for the given channel @@ -60,7 +60,7 @@ namespace eCAL * * @return true if succeeds, false if it fails **/ - bool GetEntriesInfo(const std::string& channel_name, EntryInfoSet& entries) const; + bool GetEntriesInfo(const std::string& channel_name, EntryInfoSet& entries) const override; }; } // namespace eh5 } // namespace eCAL diff --git a/contrib/ecalhdf5/src/eh5_meas_file_writer_v5.cpp b/contrib/ecalhdf5/src/eh5_meas_file_writer_v5.cpp new file mode 100644 index 0000000000..ae192ad314 --- /dev/null +++ b/contrib/ecalhdf5/src/eh5_meas_file_writer_v5.cpp @@ -0,0 +1,387 @@ +/* ========================= eCAL LICENSE ================================= + * + * Copyright (C) 2016 - 2019 Continental Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * ========================= eCAL LICENSE ================================= +*/ + +/** + * @brief eCALHDF5 directory reader +**/ + +#include "eh5_meas_file_writer_v5.h" +#include "escape.h" + +#ifdef WIN32 +#include +#else +#include +#endif //WIN32 + +#include +#include +#include + +#include +#include + +constexpr unsigned int kDefaultMaxFileSizeMB = 1000; + +eCAL::eh5::HDF5MeasFileWriterV5::HDF5MeasFileWriterV5() + : cb_pre_split_ (nullptr) + , file_id_ (-1) + , file_split_counter_(-1) + , entries_counter_ (0) + , max_size_per_file_ (kDefaultMaxFileSizeMB * 1024 * 1024) +{} + +eCAL::eh5::HDF5MeasFileWriterV5::~HDF5MeasFileWriterV5() +{ + // call the function via its class becase it's a virtual function that is called in constructor/destructor,- + // where the vtable is not created yet or it's destructed. + HDF5MeasFileWriterV5::Close(); +} + +bool eCAL::eh5::HDF5MeasFileWriterV5::Open(const std::string& output_dir, eAccessType /*access = eAccessType::RDONLY*/) +{ + Close(); + + // Check if the given path points to a directory + if (!EcalUtils::Filesystem::IsDir(output_dir, EcalUtils::Filesystem::Current)) + return false; + + output_dir_ = output_dir; + + return true; +} + +bool eCAL::eh5::HDF5MeasFileWriterV5::Close() +{ + if (!this->IsOk()) return false; + + std::string channels_with_entries; + + for (const auto& channel : channels_) + if (CreateEntriesTableOfContentsFor(channel.first, channel.second.Type, channel.second.Description, channel.second.Entries)) + channels_with_entries += channel.first + ","; + + if ((!channels_with_entries.empty()) && (channels_with_entries.back() == ',')) + channels_with_entries.pop_back(); + + SetAttribute(file_id_, kChnAttrTitle, channels_with_entries); + + for (auto& channel : channels_) + channel.second.Entries.clear(); + + if (H5Fclose(file_id_) >= 0) + { + file_id_ = -1; + return true; + } + else + { + return false; + } +} + +bool eCAL::eh5::HDF5MeasFileWriterV5::IsOk() const +{ + return (file_id_ >= 0); +} + +std::string eCAL::eh5::HDF5MeasFileWriterV5::GetFileVersion() const +{ + // UNSUPPORTED FUNCTION + return ""; +} + +size_t eCAL::eh5::HDF5MeasFileWriterV5::GetMaxSizePerFile() const +{ + return max_size_per_file_ / 1024 / 1024; +} + +void eCAL::eh5::HDF5MeasFileWriterV5::SetMaxSizePerFile(size_t size) +{ + max_size_per_file_ = size * 1024 * 1024; +} + +bool eCAL::eh5::HDF5MeasFileWriterV5::IsOneFilePerChannelEnabled() const +{ + return false; +} + +void eCAL::eh5::HDF5MeasFileWriterV5::SetOneFilePerChannelEnabled(bool /*enabled*/) +{ +} + +std::set eCAL::eh5::HDF5MeasFileWriterV5::GetChannelNames() const +{ + // UNSUPPORTED FUNCTION + return {}; +} + +bool eCAL::eh5::HDF5MeasFileWriterV5::HasChannel(const std::string& /*channel_name*/) const +{ + // UNSUPPORTED FUNCTION + return false; +} + +std::string eCAL::eh5::HDF5MeasFileWriterV5::GetChannelDescription(const std::string& /*channel_name*/) const +{ + // UNSUPPORTED FUNCTION + return ""; +} + +void eCAL::eh5::HDF5MeasFileWriterV5::SetChannelDescription(const std::string& channel_name, const std::string& description) +{ + channels_[channel_name].Description = description; +} + +std::string eCAL::eh5::HDF5MeasFileWriterV5::GetChannelType(const std::string& /*channel_name*/) const +{ + // UNSUPPORTED FUNCTION + return ""; +} + +void eCAL::eh5::HDF5MeasFileWriterV5::SetChannelType(const std::string& channel_name, const std::string& type) +{ + channels_[channel_name].Type = type; +} + +long long eCAL::eh5::HDF5MeasFileWriterV5::GetMinTimestamp(const std::string& /*channel_name*/) const +{ + // UNSUPPORTED FUNCTION + return -1; +} + +long long eCAL::eh5::HDF5MeasFileWriterV5::GetMaxTimestamp(const std::string& /*channel_name*/) const +{ + // UNSUPPORTED FUNCTION + return -1; +} + +bool eCAL::eh5::HDF5MeasFileWriterV5::GetEntriesInfo(const std::string& /*channel_name*/, EntryInfoSet& /*entries*/) const +{ + // UNSUPPORTED FUNCTION + return false; +} + +bool eCAL::eh5::HDF5MeasFileWriterV5::GetEntriesInfoRange(const std::string& /*channel_name*/, long long /*begin*/, long long /*end*/, EntryInfoSet& /*entries*/) const +{ + // UNSUPPORTED FUNCTION + return false; +} + +bool eCAL::eh5::HDF5MeasFileWriterV5::GetEntryDataSize(long long /*entry_id*/, size_t& /*size*/) const +{ + // UNSUPPORTED FUNCTION + return false; +} + +bool eCAL::eh5::HDF5MeasFileWriterV5::GetEntryData(long long /*entry_id*/, void* /*data*/) const +{ + // UNSUPPORTED FUNCTION + return false; +} + +void eCAL::eh5::HDF5MeasFileWriterV5::SetFileBaseName(const std::string& base_name) +{ + base_name_ = base_name; +} + +bool eCAL::eh5::HDF5MeasFileWriterV5::AddEntryToFile(const void* data, const unsigned long long& size, const long long& snd_timestamp, const long long& rcv_timestamp, const std::string& channel_name, long long id, long long clock) +{ + if (!IsOk()) file_id_ = Create(); + if (!IsOk()) + return false; + + hsize_t hsSize = static_cast(size); + + if (!EntryFitsTheFile(hsSize)) + { + if (cb_pre_split_ != nullptr) + { + cb_pre_split_(); + } + + if (Create() < 0) + return false; + } + + // Create DataSpace with rank 1 and size dimension + auto dataSpace = H5Screate_simple(1, &hsSize, nullptr); + + // Create creation property for dataSpace + auto dsProperty = H5Pcreate(H5P_DATASET_CREATE); + H5Pset_obj_track_times(dsProperty, false); + + // Create dataset in dataSpace + auto dataSet = H5Dcreate(file_id_, std::to_string(entries_counter_).c_str(), H5T_NATIVE_UCHAR, dataSpace, H5P_DEFAULT, dsProperty, H5P_DEFAULT); + + // Write buffer to dataset + herr_t writeStatus = H5Dwrite(dataSet, H5T_NATIVE_UCHAR, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); + + // Close dataset, data space, and data set property + H5Dclose(dataSet); + H5Pclose(dsProperty); + H5Sclose(dataSpace); + + channels_[channel_name].Entries.emplace_back(SEntryInfo(rcv_timestamp, static_cast(entries_counter_), clock, snd_timestamp, id)); + + entries_counter_++; + + return (writeStatus >= 0); +} + +void eCAL::eh5::HDF5MeasFileWriterV5::ConnectPreSplitCallback(CallbackFunction cb) +{ + cb_pre_split_ = cb; +} + +void eCAL::eh5::HDF5MeasFileWriterV5::DisconnectPreSplitCallback() +{ + cb_pre_split_ = nullptr; +} + +hid_t eCAL::eh5::HDF5MeasFileWriterV5::Create() +{ + if (output_dir_.empty()) return -1; + + if (!EcalUtils::Filesystem::IsDir(output_dir_, EcalUtils::Filesystem::OsStyle::Current) + && !EcalUtils::Filesystem::MkPath(output_dir_, EcalUtils::Filesystem::OsStyle::Current)) + return -1; + + if (base_name_.empty()) return -1; + + if (IsOk() && !Close()) return -1; + + file_split_counter_++; + + std::string filePath = output_dir_ + "/" + base_name_; + + if (file_split_counter_ > 0) + filePath += "_" + std::to_string(file_split_counter_); + + filePath += ".hdf5"; + + // create file access property + hid_t fileAccessPropery = H5Pcreate(H5P_FILE_ACCESS); + // create file create property + hid_t fileCreateProperty = H5Pcreate(H5P_FILE_CREATE); + + // Create hdf file and get file id + file_id_ = H5Fcreate(filePath.c_str(), H5F_ACC_TRUNC, fileCreateProperty, fileAccessPropery); + + if (file_id_ >= 0) + SetAttribute(file_id_, kFileVerAttrTitle, "5.0"); + else + file_split_counter_--; + + return file_id_; +} + +bool eCAL::eh5::HDF5MeasFileWriterV5::SetAttribute(const hid_t& id, const std::string& name, const std::string& value) +{ + if (id < 0) return false; + + if (H5Aexists(id, name.c_str()) > 0) + H5Adelete(id, name.c_str()); + // create scalar dataset + hid_t scalarDataset = H5Screate(H5S_SCALAR); + + // create new string data type + hid_t stringDataType = H5Tcopy(H5T_C_S1); + + // if attribute's value length exists, allocate space for it + if (value.length() > 0) + H5Tset_size(stringDataType, value.length()); + + // create attribute + hid_t attribute = H5Acreate(id, name.c_str(), stringDataType, scalarDataset, H5P_DEFAULT, H5P_DEFAULT); + + if (attribute < 0) return false; + + // write attribute value to attribute + herr_t writeStatus = H5Awrite(attribute, stringDataType, value.c_str()); + if (writeStatus < 0) return false; + + // close attribute + H5Aclose(attribute); + // close scalar dataset + H5Sclose(scalarDataset); + // close string data type + H5Tclose(stringDataType); + + return true; +} + +bool eCAL::eh5::HDF5MeasFileWriterV5::EntryFitsTheFile(const hsize_t& size) const +{ + hsize_t fileSize = 0; + bool status = GetFileSize(fileSize); + + // check if buffer fits the current file + return (status && ((fileSize + size) <= max_size_per_file_)); +} + +bool eCAL::eh5::HDF5MeasFileWriterV5::GetFileSize(hsize_t& size) const +{ + if (!IsOk()) + { + size = 0; + return false; + } + else + { + return H5Fget_filesize(file_id_, &size) >= 0; + } +} + +bool eCAL::eh5::HDF5MeasFileWriterV5::CreateEntriesTableOfContentsFor(const std::string& channelName, const std::string& channelType, const std::string& channelDescription, const EntryInfoVect& entries) const +{ + if (!IsOk()) return false; + + const size_t dataSetsSize = entries.size(); + + if (dataSetsSize == 0) return false; + + hsize_t dims[2] = { dataSetsSize, 5 }; + + // Create DataSpace with rank 2 and size dimension + auto dataSpace = H5Screate_simple(2, dims, nullptr); + + // Create creation property for dataSpace + auto dsProperty = H5Pcreate(H5P_DATASET_CREATE); + H5Pset_obj_track_times(dsProperty, false); + + auto dataSet = H5Dcreate(file_id_, channelName.c_str(), H5T_NATIVE_LLONG, dataSpace, H5P_DEFAULT, dsProperty, H5P_DEFAULT); + + if (dataSet < 0) return false; + + SetAttribute(dataSet, kChnTypeAttrTitle, channelType); + SetAttribute(dataSet, kChnDescAttrTitle, channelDescription); + + // Write buffer to dataset + herr_t writeStatus = H5Dwrite(dataSet, H5T_NATIVE_LLONG, H5S_ALL, H5S_ALL, H5P_DEFAULT, entries.data()); + if (writeStatus < 0) return false; + + // Close dataset, data space, and data set property + H5Dclose(dataSet); + H5Pclose(dsProperty); + H5Sclose(dataSpace); + + return true; +} + diff --git a/contrib/ecalhdf5/src/eh5_meas_file_writer_v5.h b/contrib/ecalhdf5/src/eh5_meas_file_writer_v5.h new file mode 100644 index 0000000000..0fce8f00a6 --- /dev/null +++ b/contrib/ecalhdf5/src/eh5_meas_file_writer_v5.h @@ -0,0 +1,344 @@ +/* ========================= eCAL LICENSE ================================= + * + * Copyright (C) 2016 - 2019 Continental Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * ========================= eCAL LICENSE ================================= +*/ + +/** + * eCALHDF5 file reader single channel +**/ + +#pragma once + +#include +#include +#include +#include + +#include "eh5_meas_impl.h" + +#include "hdf5.h" + +namespace eCAL +{ + namespace eh5 + { + class HDF5MeasFileWriterV5 : virtual public HDF5MeasImpl + { + public: + /** + * @brief Constructor + **/ + HDF5MeasFileWriterV5(); + + // Copy + HDF5MeasFileWriterV5(const HDF5MeasFileWriterV5&) = delete; + HDF5MeasFileWriterV5& operator=(const HDF5MeasFileWriterV5&) = delete; + + // Move + HDF5MeasFileWriterV5& operator=(HDF5MeasFileWriterV5&&) = default; + HDF5MeasFileWriterV5(HDF5MeasFileWriterV5&&) = default; + + /** + * @brief Destructor + **/ + ~HDF5MeasFileWriterV5() override; + + /** + * @brief Open file + * + * @param output_dir Input file path / measurement directory path + * @param access Access type (IGNORED, WILL ALWAYS OPEN READ-WRITE!) + * + * @return true if succeeds, false if it fails + **/ + bool Open(const std::string& output_dir, eAccessType access) override; + + /** + * @brief Close file + * + * @return true if succeeds, false if it fails + **/ + bool Close() override; + + /** + * @brief Checks if file/measurement is ok + * + * @return true if meas can be opened(read) or location is accessible(write), false otherwise + **/ + bool IsOk() const override; + + /** + * @brief Get the File Type Version of the current opened file + * + * @return file version + **/ + std::string GetFileVersion() const override; + + /** + * @brief Gets maximum allowed size for an individual file + * + * @return maximum size in MB + **/ + size_t GetMaxSizePerFile() const override; + + /** + * @brief Sets maximum allowed size for an individual file + * + * @param size maximum size in MB + **/ + void SetMaxSizePerFile(size_t size) override; + + /** + * @brief Whether each Channel shall be writte in its own file + * + * When enabled, data is clustered by channel and each channel is written + * to its own file. The filenames will consist of the basename and the + * channel name. + * + * @return true, if one file per channel is enabled + */ + bool IsOneFilePerChannelEnabled() const override; + + /** + * @brief Enable / disable the creation of one individual file per channel + * + * When enabled, data is clustered by channel and each channel is written + * to its own file. The filenames will consist of the basename and the + * channel name. + * + * @param enabled Whether one file shall be created per channel + */ + void SetOneFilePerChannelEnabled(bool enabled) override; + + /** + * @brief Get the available channel names of the current opened file / measurement + * + * @return channel names + **/ + std::set GetChannelNames() const override; + + /** + * @brief Check if channel exists in measurement + * + * @param channel_name name of the channel + * + * @return true if exists, false otherwise + **/ + bool HasChannel(const std::string& channel_name) const override; + + /** + * @brief Get the channel description for the given channel + * + * @param channel_name channel name + * + * @return channel description + **/ + std::string GetChannelDescription(const std::string& channel_name) const override; + + /** + * @brief Set description of the given channel + * + * @param channel_name channel name + * @param description description of the channel + **/ + void SetChannelDescription(const std::string& channel_name, const std::string& description) override; + + /** + * @brief Gets the channel type of the given channel + * + * @param channel_name channel name + * + * @return channel type + **/ + std::string GetChannelType(const std::string& channel_name) const override; + + /** + * @brief Set type of the given channel + * + * @param channel_name channel name + * @param type type of the channel + **/ + void SetChannelType(const std::string& channel_name, const std::string& type) override; + + /** + * @brief Gets minimum timestamp for specified channel + * + * @param channel_name channel name + * + * @return minimum timestamp value + **/ + long long GetMinTimestamp(const std::string& channel_name) const override; + + /** + * @brief Gets maximum timestamp for specified channel + * + * @param channel_name channel name + * + * @return maximum timestamp value + **/ + long long GetMaxTimestamp(const std::string& channel_name) const override; + + /** + * @brief Gets the header info for all data entries for the given channel + * Header = timestamp + entry id + * + * @param [in] channel_name channel name + * @param [out] entries header info for all data entries + * + * @return true if succeeds, false if it fails + **/ + bool GetEntriesInfo(const std::string& channel_name, EntryInfoSet& entries) const override; + + /** + * @brief Gets the header info for data entries for the given channel included in given time range (begin->end) + * Header = timestamp + entry id + * + * @param [in] channel_name channel name + * @param [in] begin time range begin timestamp + * @param [in] end time range end timestamp + * @param [out] entries header info for data entries in given range + * + * @return true if succeeds, false if it fails + **/ + bool GetEntriesInfoRange(const std::string& channel_name, long long begin, long long end, EntryInfoSet& entries) const override; + + /** + * @brief Gets data size of a specific entry + * + * @param [in] entry_id Entry ID + * @param [out] size Entry data size + * + * @return true if succeeds, false if it fails + **/ + bool GetEntryDataSize(long long entry_id, size_t& size) const override; + + /** + * @brief Gets data from a specific entry + * + * @param [in] entry_id Entry ID + * @param [out] data Entry data + * + * @return true if succeeds, false if it fails + **/ + bool GetEntryData(long long entry_id, void* data) const override; + + /** + * @brief Set measurement file base name + * + * @param base_name File base name. + **/ + void SetFileBaseName(const std::string& base_name) override; + + /** + * @brief Add entry to file + * + * @param data data to be added + * @param size size of the data + * @param snd_timestamp send timestamp + * @param rcv_timestamp receive timestamp + * @param channel_name channel name + * @param id message id + * @param clock message clock + * + * @return true if succeeds, false if it fails + **/ + bool AddEntryToFile(const void* data, const unsigned long long& size, const long long& snd_timestamp, const long long& rcv_timestamp, const std::string& channel_name, long long id, long long clock) override; + + using CallbackFunction = std::function; + /** + * @brief Connect callback for pre file split notification + * + * @param cb callback function + **/ + void ConnectPreSplitCallback(CallbackFunction cb) override; + + /** + * @brief Disconnect pre file split callback + **/ + void DisconnectPreSplitCallback() override; + + protected: + struct Channel + { + std::string Description; + std::string Type; + EntryInfoVect Entries; + }; + + using Channels = std::map; + + std::string output_dir_; + std::string base_name_; + Channels channels_; + CallbackFunction cb_pre_split_; + hid_t file_id_; + int file_split_counter_; + unsigned long long entries_counter_; + size_t max_size_per_file_; + + /** + * @brief Creates the actual file + * + * @return file ID, file was not created if id is negative + **/ + hid_t Create(); + + /** + * @brief Set attribute to object(file, entry...) + * + * @param id ID of the attributes parent + * @param name Name of the attribute + * @param value Value of the attribute + * + * @return true if succeeds, false if it fails + **/ + static bool SetAttribute(const hid_t& id, const std::string& name, const std::string& value); + + /** + * @brief Checks if current file size + entry size does not exceed the maximum allowed size of the file + * + * @param size Size of the entry in bytes + * + * @return true if entry can be saved in current file, false if it can not be added to the current file + **/ + bool EntryFitsTheFile(const hsize_t& size) const; + + /** + * @brief Gets the size of the file + * + * @param size Size of the file in bytes + * + * @return true if succeeds, false if it fails + **/ + bool GetFileSize(hsize_t& size) const; + + /** + * @brief Creates the entries "table of contents" (timestamp + entry id) + * (Call it just before closing the file) + * + * @param channelName name for the dataset + * @param channelType type for the dataset + * @param channelDescription description for the dataset + * @param entries entries for given channel + * + * @return true if succeeds, false if it fails + **/ + bool CreateEntriesTableOfContentsFor(const std::string& channelName, const std::string& channelType, const std::string& channelDescription, const EntryInfoVect& entries) const; + + }; + } // namespace eh5 +} // namespace eCAL diff --git a/contrib/ecalhdf5/src/eh5_meas_impl.h b/contrib/ecalhdf5/src/eh5_meas_impl.h index b7ec8b9dcc..e1825b5103 100644 --- a/contrib/ecalhdf5/src/eh5_meas_impl.h +++ b/contrib/ecalhdf5/src/eh5_meas_impl.h @@ -39,7 +39,7 @@ namespace eCAL /** * @brief Destructor **/ - virtual ~HDF5MeasImpl() {}; + virtual ~HDF5MeasImpl() = default; /** * @brief Open file @@ -86,6 +86,29 @@ namespace eCAL **/ virtual void SetMaxSizePerFile(size_t size) = 0; + /** + * @brief Whether each Channel shall be writte in its own file + * + * When enabled, data is clustered by channel and each channel is written + * to its own file. The filenames will consist of the basename and the + * channel name. + * + * @return true, if one file per channel is enabled + */ + virtual bool IsOneFilePerChannelEnabled() const = 0; + + /** + * @brief Enable / disable the creation of one individual file per channel + * + * When enabled, data is clustered by channel and each channel is written + * to its own file. The filenames will consist of the basename and the + * channel name. + * + * @param enabled Whether one file shall be created per channel + */ + virtual void SetOneFilePerChannelEnabled(bool enabled) = 0; + + /** * @brief Get the available channel names of the current opened file / measurement * diff --git a/contrib/ecalhdf5/src/escape.cpp b/contrib/ecalhdf5/src/escape.cpp index 2ea08ea166..b8f7f7320b 100644 --- a/contrib/ecalhdf5/src/escape.cpp +++ b/contrib/ecalhdf5/src/escape.cpp @@ -292,16 +292,16 @@ namespace eCAL std::string output; output.reserve(input.size() * 3 + 1); // Reserve enough so we never have to re-allocate + trailing zero (snprintf) - for (size_t i = 0; i < input.size(); i++) + for (char i : input) { - if (is_reserved_[static_cast(input[i])]) + if (is_reserved_[static_cast(i)]) { output += "%xx"; - snprintf(&output[output.size() - 2], 3, "%02X", static_cast(input[i])); + std::snprintf(&output[output.size() - 2], 3, "%02X", static_cast(i)); } else { - output += input[i]; + output += i; } } @@ -320,8 +320,8 @@ namespace eCAL if (i + 2 < input.size()) { std::string hex_string = input.substr(i + 1, 2); - unsigned int char_num; - if (sscanf(hex_string.c_str(), "%X", &char_num)) + unsigned int char_num = 0; + if (std::sscanf(hex_string.c_str(), "%X", &char_num) != 0) { output += static_cast(char_num); i += 2; diff --git a/doc/rst/applications/rec/rec_server_cli_interactive_help.txt b/doc/rst/applications/rec/rec_server_cli_interactive_help.txt index 260a92dfce..9a8a86dd6c 100644 --- a/doc/rst/applications/rec/rec_server_cli_interactive_help.txt +++ b/doc/rst/applications/rec/rec_server_cli_interactive_help.txt @@ -21,6 +21,7 @@ setconfig [--set-client ] ... | [--meas-name ] | [--max-file-size ] | [--description ] + | [--enable-one-file-per-topic ] | [--ftp-server ] | [--delete-after-upload ] | [--enable-built-in-client ] diff --git a/doc/rst/applications/rec/rec_server_cli_usage.txt b/doc/rst/applications/rec/rec_server_cli_usage.txt index d7cd656606..d40ad3981a 100644 --- a/doc/rst/applications/rec/rec_server_cli_usage.txt +++ b/doc/rst/applications/rec/rec_server_cli_usage.txt @@ -10,6 +10,7 @@ USAGE: ] [--blacklist ] [--whitelist ] [-d ] [-n ] [--max-file-size ] [--description ] [--ftp-server ] + [--enable-one-file-per-topic ] [--delete-after-upload ] [-i] [--interactive-dont-exit] [--] [--version] [-h] @@ -140,6 +141,11 @@ Where: set. If a configuration file is being loaded, this will override the setting from the config. Not available in remote-control mode. + --enable-one-file-per-topic + Whether to separate each topic in HDF5 file. This helps faster file + transfer and less network congestion in case of interest of specific + topics only. + --ftp-server The server where to upload to when uploading a measurement. Use "internal" for the integrated FTP Server. When using an external FTP diff --git a/samples/cpp/services/rec_client_service_cli/src/ecalrecorder_client.cpp b/samples/cpp/services/rec_client_service_cli/src/ecalrecorder_client.cpp index fb8d5e142f..4f42bb07e2 100644 --- a/samples/cpp/services/rec_client_service_cli/src/ecalrecorder_client.cpp +++ b/samples/cpp/services/rec_client_service_cli/src/ecalrecorder_client.cpp @@ -131,11 +131,13 @@ int main(int argc, char **argv) // state request eCAL::pb::rec_client::CommandRequest state_request; - (*state_request.mutable_command_params()->mutable_items())["meas_id"] = "1234"; // An ID to identify the measurement later on - (*state_request.mutable_command_params()->mutable_items())["meas_root_dir"] = "$TARGET{OSSELECT WIN \"C:\" LINUX \"$TARGET{ENV HOME}\"}/ecal_meas"; // The root directory to save the measurement to (un-evaluated format) - (*state_request.mutable_command_params()->mutable_items())["meas_name"] = "meas_${TIME}"; // The name of the measurement (un-evaluated format) - (*state_request.mutable_command_params()->mutable_items())["description"] = "This is my description :)"; // The description that will be saved to the measurement's doc folder (un-evaluated format). - (*state_request.mutable_command_params()->mutable_items())["max_file_size_mib"] = "775"; // The maximum HDF5 file size (When exceeding the file size, the measurement will be splitted into multiple files). + (*state_request.mutable_command_params()->mutable_items())["meas_id"] = "1234"; // An ID to identify the measurement later on + (*state_request.mutable_command_params()->mutable_items())["meas_root_dir"] = "$TARGET{OSSELECT WIN \"C:\" LINUX \"$TARGET{ENV HOME}\"}/ecal_meas"; // The root directory to save the measurement to (un-evaluated format) + (*state_request.mutable_command_params()->mutable_items())["meas_name"] = "meas_${TIME}"; // The name of the measurement (un-evaluated format) + (*state_request.mutable_command_params()->mutable_items())["description"] = "This is my description :)"; // The description that will be saved to the measurement's doc folder (un-evaluated format). + (*state_request.mutable_command_params()->mutable_items())["max_file_size_mib"] = "775"; // The maximum HDF5 file size (When exceeding the file size, the measurement will be splitted into multiple files). + (*state_request.mutable_command_params()->mutable_items())["one_file_per_topic"] = "false"; // Whether the recorder shall create 1 hdf5 file per channel + // "initialize" std::cout << "eCAL.pb.rec.EcalRecService:SetCommand()" << std::endl; diff --git a/samples/cpp/services/rec_client_service_gui/src/EcalrecGuiClient.cpp b/samples/cpp/services/rec_client_service_gui/src/EcalrecGuiClient.cpp index 83d2cb815c..e16fac7218 100644 --- a/samples/cpp/services/rec_client_service_gui/src/EcalrecGuiClient.cpp +++ b/samples/cpp/services/rec_client_service_gui/src/EcalrecGuiClient.cpp @@ -159,6 +159,11 @@ void EcalrecGuiClient::commandRequest() (*command_params)["max_file_size_mib"] = ui_.command_request_max_file_size_mib_lineedit->text().toStdString(); } + if (ui_.command_one_file_per_topic_checkbox->isChecked()) + { + (*command_params)["one_file_per_topic"] = ui_.command_one_file_per_topic_lineedit->text().toStdString(); + } + if (ui_.command_request_protocol_checkbox->isChecked()) { (*command_params)["protocol"] = ui_.command_request_protocol_lineedit->text().toStdString(); diff --git a/samples/cpp/services/rec_client_service_gui/src/MainWindow.ui b/samples/cpp/services/rec_client_service_gui/src/MainWindow.ui index 4acb378ece..57e67d9f46 100644 --- a/samples/cpp/services/rec_client_service_gui/src/MainWindow.ui +++ b/samples/cpp/services/rec_client_service_gui/src/MainWindow.ui @@ -7,7 +7,7 @@ 0 0 1024 - 707 + 731 @@ -52,7 +52,7 @@ - 0 + 2 @@ -283,17 +283,24 @@ - - + + + + meas_name + + + + + false - - + + - meas_name + meas_root_dir @@ -304,15 +311,8 @@ - - - - description - - - - - + + false @@ -325,6 +325,13 @@ + + + + description + + + @@ -332,10 +339,17 @@ - - + + - meas_root_dir + one_file_per_topic + + + + + + + false @@ -559,7 +573,6 @@ - tabWidget get_config_request_button set_config_max_pre_buffer_length_secs_checkbox set_config_max_pre_buffer_length_secs_lineedit @@ -585,6 +598,9 @@ command_request_description_textedit command_request_max_file_size_mib_checkbox command_request_max_file_size_mib_lineedit + get_state_request_button + command_one_file_per_topic_checkbox + command_one_file_per_topic_lineedit command_request_protocol_checkbox command_request_protocol_lineedit command_request_username_checkbox @@ -604,9 +620,9 @@ command_request_comment_checkbox command_request_comment_textedit command_request_button - response_texteedit response_clear_button - get_state_request_button + response_texteedit + tabWidget hostname_lineedit @@ -922,8 +938,24 @@ setEnabled(bool) - 122 - 515 + 148 + 668 + + + 405 + 670 + + + + + command_one_file_per_topic_checkbox + toggled(bool) + command_one_file_per_topic_lineedit + setEnabled(bool) + + + 102 + 371 216 diff --git a/testing/contrib/ecalhdf5/hdf5_test/src/hdf5_test.cpp b/testing/contrib/ecalhdf5/hdf5_test/src/hdf5_test.cpp index 2b1a25bfd4..9c6e112e05 100644 --- a/testing/contrib/ecalhdf5/hdf5_test/src/hdf5_test.cpp +++ b/testing/contrib/ecalhdf5/hdf5_test/src/hdf5_test.cpp @@ -90,7 +90,7 @@ TEST(HDF5, WriteReadIntegrity) const long long t2_id = 2LL; const long long t2_clock = 12LL; - const std::string t3_name = " ASCII and beyond!\a\b\t\n\v\f\r\"#$%&\'()*+,-./0123456789:;<=>\?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~"; + const std::string t3_name = " ASCII and beyond!\a\b\t\n\v\f\r\"#$%&\'()*+,-./0123456789:;<=>\?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~üöäÜÖÄâÂôÔûÛáàÁÀúÙ"; const std::string t3_data = "o.O"; const long long t3_snd_timestamp = 1003LL; const long long t3_rcv_timestamp = 2003LL; @@ -386,3 +386,139 @@ TEST(HDF5, Performance_4096kb) MeasPerf("meas_4096_pkg", DATA_SET_SIZE_4, DATA_SET_NUMBER_4); } #endif // TEST_SIZE_4 + +TEST(HDF5, IsOneFilePerChannelEnabled) +{ + eCAL::eh5::HDF5Meas hdf5_writer; + std::string base_name = "output"; + + if (hdf5_writer.Open(output_dir, eCAL::eh5::eAccessType::CREATE)) + { + hdf5_writer.SetFileBaseName(base_name); + hdf5_writer.SetMaxSizePerFile(max_size_per_file); + hdf5_writer.SetOneFilePerChannelEnabled(true); + } + else + { + FAIL() << "Failed to open HDF5 Writer"; + } + + EXPECT_TRUE(hdf5_writer.IsOneFilePerChannelEnabled()); + + hdf5_writer.SetOneFilePerChannelEnabled(false); + EXPECT_TRUE(!hdf5_writer.IsOneFilePerChannelEnabled()); +} + +TEST(HDF5, SetOneFilePerChannelEnabled) +{ + // Define data that will be written to the file + const std::string t1_name = "topic_1"; + const std::string t1_data = "topic1: test data"; + const long long t1_snd_timestamp = 1001LL; + const long long t1_rcv_timestamp = 2001LL; + const long long t1_id = 1LL; + const long long t1_clock = 11LL; + + const std::string t2_name = "topic_2"; + const std::string t2_data = "topic2: test data"; + const long long t2_snd_timestamp = 1002LL; + const long long t2_rcv_timestamp = 2002LL; + const long long t2_id = 2LL; + const long long t2_clock = 12LL; + + std::string base_name = "output"; + + // Write HDF5 file + { + eCAL::eh5::HDF5Meas hdf5_writer; + + if (hdf5_writer.Open(output_dir, eCAL::eh5::eAccessType::CREATE)) + { + hdf5_writer.SetFileBaseName(base_name); + hdf5_writer.SetMaxSizePerFile(max_size_per_file); + hdf5_writer.SetOneFilePerChannelEnabled(true); + } + else + { + FAIL() << "Failed to open HDF5 Writer"; + } + + EXPECT_TRUE(hdf5_writer.AddEntryToFile( + t1_data.data(), // data + t1_data.size(), // data size + t1_snd_timestamp, // snd_timestamp + t1_rcv_timestamp, // rcv_timestamp + t1_name, // channel name + t1_id, // id + t1_clock // clock + )); + + EXPECT_TRUE(hdf5_writer.AddEntryToFile( + t2_data.data(), // data + t2_data.size(), // data size + t2_snd_timestamp, // snd_timestamp + t2_rcv_timestamp, // rcv_timestamp + t2_name, // channel name + t2_id, // id + t2_clock // clock + )); + + EXPECT_TRUE(hdf5_writer.IsOneFilePerChannelEnabled()); + + EXPECT_TRUE(hdf5_writer.Close()); + } + + // Read HDF5 topic 1 file + { + eCAL::eh5::HDF5Meas hdf5_reader; + + EXPECT_TRUE(hdf5_reader.Open(output_dir + "/" + base_name + "_" + t1_name + ".hdf5")); + + std::set expected_channel_names {t1_name}; + EXPECT_EQ(hdf5_reader.GetChannelNames(), expected_channel_names); + + eCAL::eh5::EntryInfoSet entries_info_set_t1; + EXPECT_TRUE(hdf5_reader.GetEntriesInfo(t1_name, entries_info_set_t1)); + EXPECT_EQ(entries_info_set_t1.size(), 1); + + EXPECT_EQ(entries_info_set_t1.begin()->SndTimestamp, t1_snd_timestamp); + EXPECT_EQ(entries_info_set_t1.begin()->RcvTimestamp, t1_rcv_timestamp); + EXPECT_EQ(entries_info_set_t1.begin()->SndID, t1_id); + EXPECT_EQ(entries_info_set_t1.begin()->SndClock, t1_clock); + + size_t t1_data_size; + EXPECT_TRUE(hdf5_reader.GetEntryDataSize(entries_info_set_t1.begin()->ID, t1_data_size)); + + std::string t1_data_read(t1_data_size, ' '); + EXPECT_TRUE(hdf5_reader.GetEntryData(entries_info_set_t1.begin()->ID, const_cast(t1_data_read.data()))); + EXPECT_EQ(t1_data_read, t1_data); + } + + // Read HDF5 topic 2 file + { + eCAL::eh5::HDF5Meas hdf5_reader; + + EXPECT_TRUE(hdf5_reader.Open(output_dir + "/" + base_name + "_" + t2_name + ".hdf5")); + + std::set expected_channel_names {t2_name}; + EXPECT_EQ(hdf5_reader.GetChannelNames(), expected_channel_names); + + eCAL::eh5::EntryInfoSet entries_info_set_t2; + EXPECT_TRUE(hdf5_reader.GetEntriesInfo(t2_name, entries_info_set_t2)); + EXPECT_EQ(entries_info_set_t2.size(), 1); + + EXPECT_EQ(entries_info_set_t2.begin()->SndTimestamp, t2_snd_timestamp); + EXPECT_EQ(entries_info_set_t2.begin()->RcvTimestamp, t2_rcv_timestamp); + EXPECT_EQ(entries_info_set_t2.begin()->SndID, t2_id); + EXPECT_EQ(entries_info_set_t2.begin()->SndClock, t2_clock); + + size_t t2_data_size; + EXPECT_TRUE(hdf5_reader.GetEntryDataSize(entries_info_set_t2.begin()->ID, t2_data_size)); + + std::string t2_data_read(t2_data_size, ' '); + EXPECT_TRUE(hdf5_reader.GetEntryData(entries_info_set_t2.begin()->ID, const_cast(t2_data_read.data()))); + EXPECT_EQ(t2_data_read, t2_data); + } + +} +