eCAL Rec / ecalhdf5: Support one file per topic feature (#755)

Co-authored-by: Florian Reimold <[email protected]>
eclipse-ecal · Sep 9, 2022 · 67cf3ec · 67cf3ec
1 parent 7127b1d
commit 67cf3ec
Show file tree

Hide file tree

Showing 52 changed files with 1,648 additions and 501 deletions.
diff --git a/app/app_pb/src/ecal/app/pb/rec/client_service.proto b/app/app_pb/src/ecal/app/pb/rec/client_service.proto
@@ -54,6 +54,7 @@ message Configuration
                                           // meas_name                   [string]                  The name of the measurement (un-evaluated format)
                                           // description                 [string]                  The description that will be saved to the measurement's doc folder (un-evaluated format)
                                           // max_file_size_mib           [uint]                    The maximum HDF5 file size (When exceeding the file size, the measurement will be splitted into multiple files).
+                                          // one_file_per_topic          [bool]                    Whether the recorder shall create 1 hdf5 file per channel
 
                                           // ==== Upload measurement config ====
                                           // protocol                    [string]                  The upload type to use (e.g. ftp). More types may be added in the future, if necessary.

diff --git a/app/app_pb/src/ecal/app/pb/rec/server_config.proto b/app/app_pb/src/ecal/app/pb/rec/server_config.proto
@@ -57,6 +57,7 @@ message RecServerConfig
   string                         root_dir                   =  1;               // The root directory where to save the measurements to
   string                         meas_name                  =  2;               // The name for new measurements. This name will be used for the name of a sub-directory in the root dir.
   int64                          max_file_size_mib          =  3;               // Max file size in MiB for HDF5 files
+  bool                           one_file_per_topic         =  11;              // Whether to create 1 HDF5 file per eCAL Topic
   string                         description                =  4;               // The description for new measurements. Will be stored in the /doc/direction.txt file.
   map<string, RecClientConfig>   enabled_clients_config     =  5;               // All enabled rec clients along with their specific configuration
   bool                           pre_buffer_enabled         =  6;               // Whether pre-buffering is enabled

diff --git a/app/rec/rec_client_cli/src/ecal_rec_service.cpp b/app/rec/rec_client_cli/src/ecal_rec_service.cpp
@@ -620,6 +620,22 @@ ::eCAL::rec::JobConfig EcalRecService::ToJobConfig(const ::eCAL::pb::rec_client:
     }
   }
 
+  //////////////////////////////////////
+  // one_file_per_topic             //
+  //////////////////////////////////////
+  {
+    auto it = config.items().find("one_file_per_topic");
+    if (it != config.items().end())
+    {
+      std::string one_file_per_topic = it->second;
+      job_config.SetOneFilePerTopicEnabled(strToBool(one_file_per_topic));
+    }
+    else
+    {
+      job_config.SetOneFilePerTopicEnabled(false);
+    }
+  }
+
   //////////////////////////////////////
   // description                      //
   //////////////////////////////////////

diff --git a/app/rec/rec_client_core/include/rec_client_core/ecal_rec_defs.h b/app/rec/rec_client_core/include/rec_client_core/ecal_rec_defs.h
@@ -20,7 +20,7 @@
 #pragma once
 
 #define ECAL_REC_VERSION_MAJOR              2
-#define ECAL_REC_VERSION_MINOR              5
+#define ECAL_REC_VERSION_MINOR              6
 #define ECAL_REC_VERSION_PATCH              0
 
 #define __ECAL_REC_FUNCTION_TO_STR(x) #x                                // Stringify any input

diff --git a/app/rec/rec_client_core/include/rec_client_core/job_config.h b/app/rec/rec_client_core/include/rec_client_core/job_config.h
@@ -52,6 +52,9 @@ namespace eCAL
       void SetMaxFileSize(int64_t max_file_size_mb);
       int64_t GetMaxFileSize() const;
 
+      void SetOneFilePerTopicEnabled(bool enabled);
+      bool GetOneFilePerTopicEnabled() const;
+
       void SetDescription(const std::string& description);
       std::string GetDescription() const;
 
@@ -72,6 +75,7 @@ namespace eCAL
       std::string  meas_root_dir_;
       std::string  meas_name_;
       int64_t      max_file_size_mb_;
+      bool         one_file_per_topic_;
       std::string  description_;
     };
   }

diff --git a/app/rec/rec_client_core/src/job/hdf5_writer_thread.cpp b/app/rec/rec_client_core/src/job/hdf5_writer_thread.cpp
@@ -272,6 +272,7 @@ namespace eCAL
 
         hdf5_writer_->SetFileBaseName(host_name);
         hdf5_writer_->SetMaxSizePerFile(job_config_.GetMaxFileSize());
+        hdf5_writer_->SetOneFilePerChannelEnabled(job_config_.GetOneFilePerTopicEnabled());
       }
       else
       {

diff --git a/app/rec/rec_client_core/src/job_config.cpp b/app/rec/rec_client_core/src/job_config.cpp
@@ -33,6 +33,7 @@ namespace eCAL
     JobConfig::JobConfig()
       : job_id_(0)
       , max_file_size_mb_(50)
+      , one_file_per_topic_(false)
     {}
 
     JobConfig::~JobConfig()
@@ -62,17 +63,20 @@ namespace eCAL
       job_id_ = (((int64_t)counter) << 32) | (int64_t(0xFFFFF000) & system_clock_nsecs) | (int64_t(0x00000FFF) & steady_clock_nsecs);
     }
 
-    void            JobConfig::SetMeasRootDir   (const std::string& meas_root_dir) { meas_root_dir_ = meas_root_dir; }
-    std::string     JobConfig::GetMeasRootDir   () const                           { return meas_root_dir_; }
+    void            JobConfig::SetMeasRootDir           (const std::string& meas_root_dir) { meas_root_dir_ = meas_root_dir; }
+    std::string     JobConfig::GetMeasRootDir           () const                           { return meas_root_dir_; }
 
-    void            JobConfig::SetMeasName      (const std::string& meas_name)     { meas_name_ = meas_name; }
-    std::string     JobConfig::GetMeasName      () const                           { return meas_name_; }
+    void            JobConfig::SetMeasName              (const std::string& meas_name)     { meas_name_ = meas_name; }
+    std::string     JobConfig::GetMeasName              () const                           { return meas_name_; }
 
-    void            JobConfig::SetMaxFileSize   (int64_t max_file_size_mb)         { max_file_size_mb_ = max_file_size_mb; }
-    int64_t         JobConfig::GetMaxFileSize   () const                           { return max_file_size_mb_; }
+    void            JobConfig::SetMaxFileSize           (int64_t max_file_size_mb)         { max_file_size_mb_ = max_file_size_mb; }
+    int64_t         JobConfig::GetMaxFileSize           () const                           { return max_file_size_mb_; }
 
-    void            JobConfig::SetDescription   (const std::string& description)   { description_ = description; }
-    std::string     JobConfig::GetDescription   () const                           { return description_; }
+    void            JobConfig::SetOneFilePerTopicEnabled(bool enabled)                     { one_file_per_topic_ = enabled; }
+    bool            JobConfig::GetOneFilePerTopicEnabled() const                           { return one_file_per_topic_; }
+
+    void            JobConfig::SetDescription           (const std::string& description)   { description_ = description; }
+    std::string     JobConfig::GetDescription           () const                           { return description_; }
 
     //////////////////////////////
     // Evaluation

diff --git a/app/rec/rec_gui/src/qecalrec.cpp b/app/rec/rec_gui/src/qecalrec.cpp
@@ -560,6 +560,13 @@ void QEcalRec::setMaxFileSizeMib(unsigned int max_file_size_mib)
   emit maxFileSizeMibChangedSignal(max_file_size_mib);
 }
 
+void QEcalRec::setOneFilePerTopicEnabled(bool enabled)
+{
+  updateConfigModified(true);
+  rec_server_->SetOneFilePerTopicEnabled(enabled);
+  emit oneFilePerTopicEnabledChangedSignal(enabled);
+}
+
 void QEcalRec::setDescription(const std::string& description)
 {
   updateConfigModified(true);
@@ -922,7 +929,7 @@ bool QEcalRec::loadConfigFromFile(const std::string& path, bool omit_dialogs)
     emit measNameChangedSignal      (rec_server_->GetMeasName());
     emit maxFileSizeMibChangedSignal(rec_server_->GetMaxFileSizeMib());
     emit descriptionChangedSignal   (rec_server_->GetDescription());
-
+    emit oneFilePerTopicEnabledChangedSignal(rec_server_->GetOneFilePerTopicEnabled());
 
     emit usingBuiltInRecorderEnabledChangedSignal(rec_server_->IsUsingBuiltInRecorderEnabled());
 

diff --git a/app/rec/rec_gui/src/qecalrec.h b/app/rec/rec_gui/src/qecalrec.h
@@ -155,21 +155,23 @@ public slots:
 // Job Settings
 ////////////////////////////////////
 public slots:
-  void setMeasRootDir        (const std::string& meas_root_dir);
-  void setMeasName           (const std::string& meas_name);
-  void setMaxFileSizeMib     (unsigned int max_file_size_mib);
-  void setDescription        (const std::string& description);
+  void setMeasRootDir           (const std::string& meas_root_dir);
+  void setMeasName              (const std::string& meas_name);
+  void setMaxFileSizeMib        (unsigned int max_file_size_mib);
+  void setOneFilePerTopicEnabled(bool enabled);
+  void setDescription           (const std::string& description);
 
   std::string  measRootDir()    const;
   std::string  measName()       const;
   unsigned int maxFileSizeMib() const;
   std::string  description()    const;
 
 signals:
-  void measRootDirChangedSignal   (std::string  meas_root_dir);
-  void measNameChangedSignal      (std::string  meas_name);
-  void maxFileSizeMibChangedSignal(unsigned int max_file_size_mib);
-  void descriptionChangedSignal   (std::string  description);
+  void measRootDirChangedSignal           (std::string  meas_root_dir);
+  void measNameChangedSignal              (std::string  meas_name);
+  void maxFileSizeMibChangedSignal        (unsigned int max_file_size_mib);
+  void oneFilePerTopicEnabledChangedSignal(bool enabled);
+  void descriptionChangedSignal           (std::string  description);
 
 ////////////////////////////////////
 // Server Settings

diff --git a/app/rec/rec_gui/src/widgets/config_widget/config_widget.cpp b/app/rec/rec_gui/src/widgets/config_widget/config_widget.cpp
@@ -30,17 +30,19 @@ ConfigWidget::ConfigWidget(QWidget *parent)
 {
   ui_.setupUi(this);
 
-  connect(ui_.measurement_directory_lineedit, &QLineEdit::textChanged, QEcalRec::instance(), [this]() {QEcalRec::instance()->setMeasRootDir(ui_.measurement_directory_lineedit->text().toStdString()); });
-  connect(ui_.measurement_name_lineedit,      &QLineEdit::textChanged, QEcalRec::instance(), [this]() {QEcalRec::instance()->setMeasName   (ui_.measurement_name_lineedit->text().toStdString()); });
-  connect(ui_.description_textedit,           &QTextEdit::textChanged, QEcalRec::instance(), [this]() {QEcalRec::instance()->setDescription(ui_.description_textedit->toPlainText().toStdString()); });
+  connect(ui_.measurement_directory_lineedit, &QLineEdit::textChanged,  QEcalRec::instance(), [this]() {QEcalRec::instance()->setMeasRootDir(ui_.measurement_directory_lineedit->text().toStdString()); });
+  connect(ui_.measurement_name_lineedit,      &QLineEdit::textChanged,  QEcalRec::instance(), [this]() {QEcalRec::instance()->setMeasName   (ui_.measurement_name_lineedit->text().toStdString()); });
+  connect(ui_.description_textedit,           &QTextEdit::textChanged,  QEcalRec::instance(), [this]() {QEcalRec::instance()->setDescription(ui_.description_textedit->toPlainText().toStdString()); });
   connect(ui_.max_file_size_spinbox, static_cast<void (QSpinBox:: *)(int)>(&QSpinBox::valueChanged), QEcalRec::instance(), [](int megabytes) {QEcalRec::instance()->setMaxFileSizeMib(megabytes); });
+  connect(ui_.one_file_per_topic_checkbox,    &QCheckBox::stateChanged, QEcalRec::instance(), [this]() {QEcalRec::instance()->setOneFilePerTopicEnabled(ui_.one_file_per_topic_checkbox->isChecked()); });
 
   connect(ui_.refresh_path_preview_button,    &QAbstractButton::clicked,            QEcalRec::instance(), [this]() { updatePathPreviewAndWarningLabel(); });
 
   connect(QEcalRec::instance(), &QEcalRec::measRootDirChangedSignal,    this, &ConfigWidget::measurementRootDirectoryChanged);
   connect(QEcalRec::instance(), &QEcalRec::measNameChangedSignal,       this, &ConfigWidget::measurementNameChanged);
   connect(QEcalRec::instance(), &QEcalRec::maxFileSizeMibChangedSignal, this, &ConfigWidget::maxFileSizeChanged);
   connect(QEcalRec::instance(), &QEcalRec::descriptionChangedSignal,    this, &ConfigWidget::descriptionChanged);
+  connect(QEcalRec::instance(), &QEcalRec::oneFilePerTopicEnabledChangedSignal,    this, &ConfigWidget::oneFilePerTopicEnabledChanged);
 
   measurementRootDirectoryChanged (QEcalRec::instance()->measRootDir());
   measurementNameChanged          (QEcalRec::instance()->measName());
@@ -63,6 +65,15 @@ void ConfigWidget::maxFileSizeChanged(unsigned int megabytes)
   }
 }
 
+void ConfigWidget::oneFilePerTopicEnabledChanged(bool enabled)
+{
+  if (enabled != ui_.one_file_per_topic_checkbox->isChecked())
+  {
+    ui_.one_file_per_topic_checkbox->blockSignals(true);
+    ui_.one_file_per_topic_checkbox->setChecked(enabled);
+    ui_.one_file_per_topic_checkbox->blockSignals(false);
+  }
+}
 void ConfigWidget::measurementRootDirectoryChanged(const std::string& root_dir)
 {
   if (root_dir.c_str() != ui_.measurement_directory_lineedit->text())

diff --git a/app/rec/rec_gui/src/widgets/config_widget/config_widget.h b/app/rec/rec_gui/src/widgets/config_widget/config_widget.h
@@ -35,6 +35,7 @@ private slots:
   void measurementNameChanged          (const std::string& name);
   void updatePathPreviewAndWarningLabel();
   void maxFileSizeChanged              (unsigned int megabytes);
+  void oneFilePerTopicEnabledChanged   (bool enabled);
   void descriptionChanged              (const std::string& description);
 
 protected:
-Original file line number
+Diff line change
@@ Expand Up / @@ -272,6 +272,7 @@ namespace eCAL @@
             hdf5_writer_->SetFileBaseName(host_name);
             hdf5_writer_->SetMaxSizePerFile(job_config_.GetMaxFileSize());
+            hdf5_writer_->SetOneFilePerChannelEnabled(job_config_.GetOneFilePerTopicEnabled());
           }
           else
           {
@@ Expand Down @@