Move ElectrodesTable creation

docs/pages/userdocs/workflow.dox

@@ -14,12 +14,14 @@
  *    used for managing \ref AQNWB::NWB::Container "Container" objects for storing recordings.
  * 3. Create the \ref AQNWB::NWB::NWBFile "NWBFile" object used for managing and creating NWB
  *    file contents.
- * 4. Create the \ref AQNWB::NWB::Container "Container" objects (e.g.,
+ * 4. Create the \ref AQNWB::NWB::ElectrodesTable "ElectrodesTable" object used for saving 
+ *    electrodes information. 
+ * 5. Create the \ref AQNWB::NWB::Container "Container" objects (e.g.,
  *    \ref AQNWB::NWB::ElectricalSeries "ElectricalSeries") used for recording and add them
  *    to the \ref AQNWB::NWB::RecordingContainers "RecordingContainers".
- * 5. Start the recording.
- * 6. Write data.
- * 7. Stop the recording and close the \ref AQNWB::NWB::NWBFile "NWBFile".
+ * 6. Start the recording.
+ * 7. Write data.
+ * 8. Stop the recording and close the \ref AQNWB::NWB::NWBFile "NWBFile".
  *
  * Below, we walk through these steps in more detail.
  *
@@ -52,7 +54,22 @@
  * \snippet tests/examples/testWorkflowExamples.cpp example_workflow_nwbfile_snippet
  *
  *
- * \subsection create_datasets 4. Create datasets and add to RecordingContainers.
+ * \subsection create_nwbfile 4. Create the ElectrodesTable object.
+ *
+ * Next, constructs the \ref AQNWB::NWB::ElectrodesTable "ElectrodesTable" object, 
+ * using the `recordingArrays` as an input. Note that in many cases, you will be recording from 
+ * all electrodes detected by the device and this `recordingArrays` structure will be the same as
+ * is used to create the \ref AQNWB::NWB::ElectricalSeries "ElectricalSeries". However in some cases
+ * (e.g. Neuropixels), you may be recording from a subset of all available electrodes. In this case,
+ * you should create the ref AQNWB::NWB::ElectrodesTable "ElectrodesTable" with all electrodes
+ * detected by the acquisition system, not just those being actively recorded from. This approach is
+ * critical for mapping information about which \ref AQNWB::NWB::ElectricalSeries "ElectricalSeries"
+ * were recorded from which electrodes.
+ *
+ * \snippet tests/examples/testWorkflowExamples.cpp example_workflow_electrodes_table_snippet
+ * 
+ *
+ * \subsection create_datasets 5. Create datasets and add to RecordingContainers.
  *
  * Next, create the different data types (e.g. \ref AQNWB::NWB::ElectricalSeries "ElectricalSeries"
  * or other AQNWB::NWB::TimeSeries "TimeSeries") that you would like to write data into. After
@@ -67,7 +84,7 @@
  * \snippet tests/examples/testWorkflowExamples.cpp example_workflow_datasets_snippet
  *
  *
- * \subsection start_recording 5. Start the recording.
+ * \subsection start_recording 6. Start the recording.
  *
  * Then, start the recording process with a call to the ``startRecording`` function of the I/O object.
  *
@@ -81,7 +98,7 @@
  * \snippet tests/examples/testWorkflowExamples.cpp example_workflow_start_snippet
  *
  *
- * \subsection write_data 6. Write data.
+ * \subsection write_data 7. Write data.
  *
  * During the recording process, use the \ref AQNWB::NWB::RecordingContainers "RecordingContainers"
  * as an interface to access the various  \ref AQNWB::NWB::Container "Container" object and corresponding
@@ -91,7 +108,7 @@
  * \snippet tests/examples/testWorkflowExamples.cpp example_workflow_write_snippet
  *
  *
- * \subsection stop_recording 7. Stop the recording and finalize the file.
+ * \subsection stop_recording 8. Stop the recording and finalize the file.
  *
  * When the recording process is finished, call `stopRecording` from the I/O object
  * to flush any data and close the file.

src/io/BaseIO.hpp

@@ -441,6 +441,13 @@ class BaseIO
   virtual std::unique_ptr<BaseRecordingData> getDataSet(
       const std::string& path) = 0;
 
+  /**
+   * @brief Returns the size of the dataset for each dimension.
+   * @param path The location in the file of the dataset.
+   * @return The dataset size.
+   */
+  virtual std::vector<SizeType> getDatasetSize(const std::string path) = 0;
+
   /**
    * @brief Convenience function for creating NWB related attributes.
    * @param path The location of the object in the file.

src/io/hdf5/HDF5IO.cpp

@@ -1106,6 +1106,17 @@ HDF5IO::getStorageObjects(const std::string& path,
   return objects;
 }
 
+std::vector<SizeType> HDF5IO::getDatasetSize(const std::string path)
+{
+  H5::DataSet dataset = m_file->openDataSet(path);
+  H5::DataSpace dataspace = dataset.getSpace();
+  const int rank = dataspace.getSimpleExtentNdims();
+  std::vector<hsize_t> dims(static_cast<size_t>(rank));
+  dataspace.getSimpleExtentDims(dims.data());
+
+  return std::vector<SizeType>(dims.begin(), dims.end());
+}
+
 std::unique_ptr<AQNWB::IO::BaseRecordingData> HDF5IO::getDataSet(
     const std::string& path)
 {

src/io/hdf5/HDF5IO.hpp

@@ -268,6 +268,13 @@ class HDF5IO : public BaseIO
   std::unique_ptr<IO::BaseRecordingData> getDataSet(
       const std::string& path) override;
 
+  /**
+   * @brief Returns the size of the dataset for each dimension.
+   * @param path The location in the file of the dataset.
+   * @return The dataset size.
+   */
+  std::vector<SizeType> getDatasetSize(const std::string path) override;
+
   /**
    * @brief Checks whether a Dataset, Group, or Link already exists at the
    * location in the file.

src/nwb/NWBFile.cpp

@@ -28,7 +28,7 @@ constexpr SizeType CHUNK_XSIZE =
 constexpr SizeType SPIKE_CHUNK_XSIZE =
     8;  // TODO - replace with io settings input
 
-std::vector<SizeType> NWBFile::emptyContainerIndexes = {};
+std::vector<SizeType> NWBFile::m_emptyContainerIndexes = {};
 
 // Initialize the static registered_ member to trigger registration
 REGISTER_SUBCLASS_IMPL(NWBFile)
@@ -172,6 +172,43 @@ Status NWBFile::createFileStructure(const std::string& identifierText,
   return Status::Success;
 }
 
+Status NWBFile::createElectrodesTable(
+    std::vector<Types::ChannelVector> recordingArrays)
+{
+  std::unique_ptr<NWB::ElectrodeTable> electrodeTable =
+      std::make_unique<NWB::ElectrodeTable>(m_io);
+  electrodeTable->initialize();
+  for (const auto& channelVector : recordingArrays) {
+    electrodeTable->addElectrodes(channelVector);
+  }
+
+  for (size_t i = 0; i < recordingArrays.size(); ++i) {
+    const auto& channelVector = recordingArrays[i];
+
+    // Setup electrodes and devices
+    std::string groupName = channelVector[0].getGroupName();
+    std::string devicePath = AQNWB::mergePaths("/general/devices", groupName);
+    std::string electrodePath =
+        AQNWB::mergePaths("/general/extracellular_ephys", groupName);
+
+    // Check if device exists for groupName, create device and electrode group
+    // if it does not
+    if (!m_io->objectExists(devicePath)) {
+      NWB::Device device = NWB::Device(devicePath, m_io);
+      device.initialize("description", "unknown");
+
+      NWB::ElectrodeGroup elecGroup = NWB::ElectrodeGroup(electrodePath, m_io);
+      elecGroup.initialize("description", "unknown", device);
+    }
+  }
+
+  // write electrodes information to datasets
+  // (requires that ElectrodeGroup data is written)
+  electrodeTable->finalize();
+
+  return Status::Success;
+}
+
 Status NWBFile::createElectricalSeries(
     std::vector<Types::ChannelVector> recordingArrays,
     std::vector<std::string> recordingNames,
@@ -191,58 +228,36 @@ Status NWBFile::createElectricalSeries(
   bool electrodeTableCreated =
       m_io->objectExists(ElectrodeTable::electrodeTablePath);
   if (!electrodeTableCreated) {
-    m_electrodeTable = std::make_unique<ElectrodeTable>(m_io);
-    m_electrodeTable->initialize();
-
-    // Add electrode information to table (does not write to datasets yet)
-    for (const auto& channelVector : recordingArrays) {
-      m_electrodeTable->addElectrodes(channelVector);
-    }
+    std::cerr << "NWBFile::createElectricalSeries requires an electrodes table "
+                 "to be present"
+              << std::endl;
+    return Status::Failure;
   }
 
   // Create datasets
   for (size_t i = 0; i < recordingArrays.size(); ++i) {
     const auto& channelVector = recordingArrays[i];
     const std::string& recordingName = recordingNames[i];
-
-    // Setup electrodes and devices
-    std::string groupName = channelVector[0].getGroupName();
-    std::string devicePath = AQNWB::mergePaths("/general/devices", groupName);
-    std::string electrodePath =
-        AQNWB::mergePaths("/general/extracellular_ephys", groupName);
     std::string electricalSeriesPath =
-        AQNWB::mergePaths(acquisitionPath, recordingName);
-
-    // Check if device exists for groupName, create device and electrode group
-    // if it does not
-    if (!m_io->objectExists(devicePath)) {
-      Device device = Device(devicePath, m_io);
-      device.initialize("description", "unknown");
-
-      ElectrodeGroup elecGroup = ElectrodeGroup(electrodePath, m_io);
-      elecGroup.initialize("description", "unknown", device);
-    }
+        AQNWB::mergePaths(m_acquisitionPath, recordingName);
 
     // Setup electrical series datasets
     auto electricalSeries =
         std::make_unique<ElectricalSeries>(electricalSeriesPath, m_io);
-    electricalSeries->initialize(
+    Status esStatus = electricalSeries->initialize(
         dataType,
         channelVector,
         "Stores continuously sampled voltage data from an "
         "extracellular ephys recording",
         SizeArray {0, channelVector.size()},
         SizeArray {CHUNK_XSIZE, 0});
+    if (esStatus != Status::Success) {
+      return esStatus;
+    }
     recordingContainers->addContainer(std::move(electricalSeries));
     containerIndexes.push_back(recordingContainers->size() - 1);
   }
 
-  // write electrode information to datasets
-  // (requires that the ElectrodeGroup has been written)
-  if (!electrodeTableCreated) {
-    m_electrodeTable->finalize();
-  }
-
   return Status::Success;
 }
 
@@ -265,13 +280,10 @@ Status NWBFile::createSpikeEventSeries(
   bool electrodeTableCreated =
       m_io->objectExists(ElectrodeTable::electrodeTablePath);
   if (!electrodeTableCreated) {
-    m_electrodeTable = std::make_unique<ElectrodeTable>(m_io);
-    m_electrodeTable->initialize();
-
-    // Add electrode information to table (does not write to datasets yet)
-    for (const auto& channelVector : recordingArrays) {
-      m_electrodeTable->addElectrodes(channelVector);
-    }
+    std::cerr << "NWBFile::createElectricalSeries requires an electrodes table "
+                 "to be present"
+              << std::endl;
+    return Status::Failure;
   }
 
   // Create datasets
@@ -285,7 +297,7 @@ Status NWBFile::createSpikeEventSeries(
     std::string electrodePath =
         AQNWB::mergePaths("/general/extracellular_ephys", groupName);
     std::string spikeEventSeriesPath =
-        AQNWB::mergePaths(acquisitionPath, recordingName);
+        AQNWB::mergePaths(m_acquisitionPath, recordingName);
 
     // Check if device exists for groupName, create device and electrode group
     // if not
@@ -320,12 +332,6 @@ Status NWBFile::createSpikeEventSeries(
     containerIndexes.push_back(recordingContainers->size() - 1);
   }
 
-  // write electrode information to datasets
-  // (requires that the ElectrodeGroup has been written)
-  if (!electrodeTableCreated) {
-    m_electrodeTable->finalize();
-  }
-
   return Status::Success;
 }
 
@@ -341,7 +347,7 @@ Status NWBFile::createAnnotationSeries(std::vector<std::string> recordingNames,
     const std::string& recordingName = recordingNames[i];
 
     std::string annotationSeriesPath =
-        AQNWB::mergePaths(acquisitionPath, recordingName);
+        AQNWB::mergePaths(m_acquisitionPath, recordingName);
 
     // Setup annotation series datasets
     auto annotationSeries =

src/nwb/NWBFile.hpp

@@ -92,6 +92,20 @@ class NWBFile : public Container
    */
   Status finalize();
 
+  /**
+   * @brief Create ElectrodesTable.
+   * Note, this function will fail if the file is in a mode where
+   * new objects cannot be added, which can be checked via
+   * nwbfile.io->canModifyObjects()
+   * @param recordingArrays vector of ChannelVector indicating the electrodes to
+   *                        add to the table. This vector should contain all the
+   *                        electrodes that are detected by the acquisition
+   * system, not only those being actively recorded from.
+   * @return Status The status of the object creation operation.
+   */
+  Status createElectrodesTable(
+      std::vector<Types::ChannelVector> recordingArrays);
+
   /**
    * @brief Create ElectricalSeries objects to record data into.
    * Created objects are stored in recordingContainers.
@@ -114,7 +128,7 @@ class NWBFile : public Container
       std::vector<std::string> recordingNames,
       const IO::BaseDataType& dataType = IO::BaseDataType::I16,
       RecordingContainers* recordingContainers = nullptr,
-      std::vector<SizeType>& containerIndexes = emptyContainerIndexes);
+      std::vector<SizeType>& containerIndexes = m_emptyContainerIndexes);
 
   /**
    * @brief Create SpikeEventSeries objects to record data into.
@@ -135,7 +149,7 @@ class NWBFile : public Container
       std::vector<std::string> recordingNames,
       const IO::BaseDataType& dataType = IO::BaseDataType::I16,
       RecordingContainers* recordingContainers = nullptr,
-      std::vector<SizeType>& containerIndexes = emptyContainerIndexes);
+      std::vector<SizeType>& containerIndexes = m_emptyContainerIndexes);
 
   /** @brief Create AnnotationSeries objects to record data into.
    * Created objects are stored in recordingContainers.
@@ -149,7 +163,7 @@ class NWBFile : public Container
   Status createAnnotationSeries(
       std::vector<std::string> recordingNames,
       RecordingContainers* recordingContainers = nullptr,
-      std::vector<SizeType>& containerIndexes = emptyContainerIndexes);
+      std::vector<SizeType>& containerIndexes = m_emptyContainerIndexes);
 
   DEFINE_REGISTERED_FIELD(readElectrodeTable,
                           ElectrodeTable,
@@ -246,10 +260,9 @@ class NWBFile : public Container
       const std::array<std::pair<std::string_view, std::string_view>, N>&
           specVariables);
 
-  inline const static std::string acquisitionPath = "/acquisition";
-  static std::vector<SizeType> emptyContainerIndexes;
+  inline const static std::string m_acquisitionPath = "/acquisition";
+  static std::vector<SizeType> m_emptyContainerIndexes;
 
-private:
   /**
    * @brief The ElectrodeTable for the file
    */