Move ElectrodesTable creation

docs/pages/userdocs/workflow.dox

@@ -14,12 +14,15 @@
  *    used for managing \ref AQNWB::NWB::Container "Container" objects for storing recordings.
  * 3. Create the \ref AQNWB::NWB::NWBFile "NWBFile" object used for managing and creating NWB
  *    file contents.
- * 4. Create the \ref AQNWB::NWB::Container "Container" objects (e.g.,
+ * 4. Create the recording metadata stored in ``\general`` in NWB, e.g., the  
+ *    \ref AQNWB::NWB::ElectrodesTable "ElectrodesTable" object used for 
+ *    saving electrodes information. 
+ * 5. Create the \ref AQNWB::NWB::Container "Container" objects (e.g.,
  *    \ref AQNWB::NWB::ElectricalSeries "ElectricalSeries") used for recording and add them
  *    to the \ref AQNWB::NWB::RecordingContainers "RecordingContainers".
- * 5. Start the recording.
- * 6. Write data.
- * 7. Stop the recording and close the \ref AQNWB::NWB::NWBFile "NWBFile".
+ * 6. Start the recording.
+ * 7. Write data.
+ * 8. Stop the recording and close the \ref AQNWB::NWB::NWBFile "NWBFile".
  *
  * Below, we walk through these steps in more detail.
  *
@@ -52,7 +55,23 @@
  * \snippet tests/examples/testWorkflowExamples.cpp example_workflow_nwbfile_snippet
  *
  *
- * \subsection create_datasets 4. Create datasets and add to RecordingContainers.
+ * \subsection create_recmeta 4. Create the recording metadata 
+ * \subsubsection create_recmeta_ecephys Create the extracellular recording metadata
+ * Next, construct the \ref AQNWB::NWB::ElectrodesTable "ElectrodesTable" object, 
+ * using the `recordingArrays` as an input. Note that in many cases, you will be recording from 
+ * all electrodes detected by the device and this `recordingArrays` structure will be the same as
+ * is used to create the \ref AQNWB::NWB::ElectricalSeries "ElectricalSeries". However in some cases
+ * (e.g. when using Neuropixels or multiple probes), you may be recording from a subset of 
+ * all available electrodes as part of a single \ref AQNWB::NWB::ElectricalSeries "ElectricalSeries". In this case,
+ * you should create the ref AQNWB::NWB::ElectrodesTable "ElectrodesTable" with all electrodes
+ * detected by the acquisition system, not just those being actively recorded from. This approach is
+ * critical for mapping information about which \ref AQNWB::NWB::ElectricalSeries "ElectricalSeries"
+ * were recorded from which electrodes.
+ *
+ * \snippet tests/examples/testWorkflowExamples.cpp example_workflow_electrodes_table_snippet
+ * 
+ *
+ * \subsection create_datasets 5. Create datasets and add to RecordingContainers.
  *
  * Next, create the different data types (e.g. \ref AQNWB::NWB::ElectricalSeries "ElectricalSeries"
  * or other AQNWB::NWB::TimeSeries "TimeSeries") that you would like to write data into. After
@@ -67,7 +86,7 @@
  * \snippet tests/examples/testWorkflowExamples.cpp example_workflow_datasets_snippet
  *
  *
- * \subsection start_recording 5. Start the recording.
+ * \subsection start_recording 6. Start the recording.
  *
  * Then, start the recording process with a call to the ``startRecording`` function of the I/O object.
  *
@@ -81,7 +100,7 @@
  * \snippet tests/examples/testWorkflowExamples.cpp example_workflow_start_snippet
  *
  *
- * \subsection write_data 6. Write data.
+ * \subsection write_data 7. Write data.
  *
  * During the recording process, use the \ref AQNWB::NWB::RecordingContainers "RecordingContainers"
  * as an interface to access the various  \ref AQNWB::NWB::Container "Container" object and corresponding
@@ -91,7 +110,7 @@
  * \snippet tests/examples/testWorkflowExamples.cpp example_workflow_write_snippet
  *
  *
- * \subsection stop_recording 7. Stop the recording and finalize the file.
+ * \subsection stop_recording 8. Stop the recording and finalize the file.
  *
  * When the recording process is finished, call `stopRecording` from the I/O object
  * to flush any data and close the file.

src/io/BaseIO.hpp

@@ -441,6 +441,14 @@ class BaseIO
   virtual std::unique_ptr<BaseRecordingData> getDataSet(
       const std::string& path) = 0;
 
+  /**
+   * @brief Returns the size of the dataset for each dimension.
+   * @param path The location in the file of the dataset.
+   * @return The dataset shape.
+   */
+  virtual std::vector<SizeType> getStorageObjectShape(
+      const std::string path) = 0;
+
   /**
    * @brief Convenience function for creating NWB related attributes.
    * @param path The location of the object in the file.

src/io/hdf5/HDF5IO.cpp

@@ -653,16 +653,17 @@ Status HDF5IO::createAttribute(const IO::BaseDataType& type,
   H5type = getH5Type(type);
   origType = getNativeType(type);
 
+  DataSpace attr_dataspace;
   if (size > 1) {
     hsize_t dims = static_cast<hsize_t>(size);
-    H5type = ArrayType(H5type, 1, &dims);
-    origType = ArrayType(origType, 1, &dims);
+    attr_dataspace = DataSpace(1, &dims);  // Create 1D dataspace of size 'dims'
+  } else {
+    attr_dataspace = H5S_SCALAR;
   }
 
   if (loc->attrExists(name)) {
     attr = loc->openAttribute(name);
   } else {
-    DataSpace attr_dataspace(H5S_SCALAR);
     attr = loc->createAttribute(name, H5type, attr_dataspace);
   }
 
@@ -1106,6 +1107,24 @@ HDF5IO::getStorageObjects(const std::string& path,
   return objects;
 }
 
+std::vector<SizeType> HDF5IO::getStorageObjectShape(const std::string path)
+{
+  H5::DataSpace dataspace;
+  try {
+    H5::DataSet dataset = m_file->openDataSet(path);
+    dataspace = dataset.getSpace();
+  } catch (H5::Exception& e) {
+    // Read the attribute
+    std::unique_ptr<H5::Attribute> attributePtr = this->getAttribute(path);
+    dataspace = attributePtr->getSpace();
+  }
+  const int rank = dataspace.getSimpleExtentNdims();
+  std::vector<hsize_t> dims(static_cast<size_t>(rank));
+  dataspace.getSimpleExtentDims(dims.data());
+
+  return std::vector<SizeType>(dims.begin(), dims.end());
+}
+
 std::unique_ptr<AQNWB::IO::BaseRecordingData> HDF5IO::getDataSet(
     const std::string& path)
 {

src/io/hdf5/HDF5IO.hpp

@@ -268,6 +268,13 @@ class HDF5IO : public BaseIO
   std::unique_ptr<IO::BaseRecordingData> getDataSet(
       const std::string& path) override;
 
+  /**
+   * @brief Returns the size of the dataset for each dimension.
+   * @param path The location in the file of the dataset.
+   * @return The dataset shape.
+   */
+  std::vector<SizeType> getStorageObjectShape(const std::string path) override;
+
   /**
    * @brief Checks whether a Dataset, Group, or Link already exists at the
    * location in the file.

src/nwb/NWBFile.cpp

@@ -28,8 +28,6 @@ constexpr SizeType CHUNK_XSIZE =
 constexpr SizeType SPIKE_CHUNK_XSIZE =
     8;  // TODO - replace with io settings input
 
-std::vector<SizeType> NWBFile::emptyContainerIndexes = {};
-
 // Initialize the static registered_ member to trigger registration
 REGISTER_SUBCLASS_IMPL(NWBFile)
 
@@ -172,6 +170,43 @@ Status NWBFile::createFileStructure(const std::string& identifierText,
   return Status::Success;
 }
 
+Status NWBFile::createElectrodesTable(
+    std::vector<Types::ChannelVector> recordingArrays)
+{
+  std::unique_ptr<NWB::ElectrodeTable> electrodeTable =
+      std::make_unique<NWB::ElectrodeTable>(m_io);
+  electrodeTable->initialize();
+  for (const auto& channelVector : recordingArrays) {
+    electrodeTable->addElectrodes(channelVector);
+  }
+
+  for (size_t i = 0; i < recordingArrays.size(); ++i) {
+    const auto& channelVector = recordingArrays[i];
+
+    // Setup electrodes and devices
+    std::string groupName = channelVector[0].getGroupName();
+    std::string devicePath = AQNWB::mergePaths("/general/devices", groupName);
+    std::string electrodePath =
+        AQNWB::mergePaths("/general/extracellular_ephys", groupName);
+
+    // Check if device exists for groupName, create device and electrode group
+    // if it does not
+    if (!m_io->objectExists(devicePath)) {
+      NWB::Device device = NWB::Device(devicePath, m_io);
+      device.initialize("description", "unknown");
+
+      NWB::ElectrodeGroup elecGroup = NWB::ElectrodeGroup(electrodePath, m_io);
+      elecGroup.initialize("description", "unknown", device);
+    }
+  }
+
+  // write electrodes information to datasets
+  // (requires that ElectrodeGroup data is written)
+  electrodeTable->finalize();
+
+  return Status::Success;
+}
+
 Status NWBFile::createElectricalSeries(
     std::vector<Types::ChannelVector> recordingArrays,
     std::vector<std::string> recordingNames,
@@ -191,58 +226,36 @@ Status NWBFile::createElectricalSeries(
   bool electrodeTableCreated =
       m_io->objectExists(ElectrodeTable::electrodeTablePath);
   if (!electrodeTableCreated) {
-    m_electrodeTable = std::make_unique<ElectrodeTable>(m_io);
-    m_electrodeTable->initialize();
-
-    // Add electrode information to table (does not write to datasets yet)
-    for (const auto& channelVector : recordingArrays) {
-      m_electrodeTable->addElectrodes(channelVector);
-    }
+    std::cerr << "NWBFile::createElectricalSeries requires an electrodes table "
+                 "to be present"
+              << std::endl;
+    return Status::Failure;
   }
 
   // Create datasets
   for (size_t i = 0; i < recordingArrays.size(); ++i) {
     const auto& channelVector = recordingArrays[i];
     const std::string& recordingName = recordingNames[i];
-
-    // Setup electrodes and devices
-    std::string groupName = channelVector[0].getGroupName();
-    std::string devicePath = AQNWB::mergePaths("/general/devices", groupName);
-    std::string electrodePath =
-        AQNWB::mergePaths("/general/extracellular_ephys", groupName);
     std::string electricalSeriesPath =
-        AQNWB::mergePaths(acquisitionPath, recordingName);
-
-    // Check if device exists for groupName, create device and electrode group
-    // if it does not
-    if (!m_io->objectExists(devicePath)) {
-      Device device = Device(devicePath, m_io);
-      device.initialize("description", "unknown");
-
-      ElectrodeGroup elecGroup = ElectrodeGroup(electrodePath, m_io);
-      elecGroup.initialize("description", "unknown", device);
-    }
+        AQNWB::mergePaths(m_acquisitionPath, recordingName);
 
     // Setup electrical series datasets
     auto electricalSeries =
         std::make_unique<ElectricalSeries>(electricalSeriesPath, m_io);
-    electricalSeries->initialize(
+    Status esStatus = electricalSeries->initialize(
         dataType,
         channelVector,
         "Stores continuously sampled voltage data from an "
         "extracellular ephys recording",
         SizeArray {0, channelVector.size()},
         SizeArray {CHUNK_XSIZE, 0});
+    if (esStatus != Status::Success) {
+      return esStatus;
+    }
     recordingContainers->addContainer(std::move(electricalSeries));
     containerIndexes.push_back(recordingContainers->size() - 1);
   }
 
-  // write electrode information to datasets
-  // (requires that the ElectrodeGroup has been written)
-  if (!electrodeTableCreated) {
-    m_electrodeTable->finalize();
-  }
-
   return Status::Success;
 }
 
@@ -265,13 +278,10 @@ Status NWBFile::createSpikeEventSeries(
   bool electrodeTableCreated =
       m_io->objectExists(ElectrodeTable::electrodeTablePath);
   if (!electrodeTableCreated) {
-    m_electrodeTable = std::make_unique<ElectrodeTable>(m_io);
-    m_electrodeTable->initialize();
-
-    // Add electrode information to table (does not write to datasets yet)
-    for (const auto& channelVector : recordingArrays) {
-      m_electrodeTable->addElectrodes(channelVector);
-    }
+    std::cerr << "NWBFile::createElectricalSeries requires an electrodes table "
+                 "to be present"
+              << std::endl;
+    return Status::Failure;
   }
 
   // Create datasets
@@ -285,7 +295,7 @@ Status NWBFile::createSpikeEventSeries(
     std::string electrodePath =
         AQNWB::mergePaths("/general/extracellular_ephys", groupName);
     std::string spikeEventSeriesPath =
-        AQNWB::mergePaths(acquisitionPath, recordingName);
+        AQNWB::mergePaths(m_acquisitionPath, recordingName);
 
     // Check if device exists for groupName, create device and electrode group
     // if not
@@ -320,12 +330,6 @@ Status NWBFile::createSpikeEventSeries(
     containerIndexes.push_back(recordingContainers->size() - 1);
   }
 
-  // write electrode information to datasets
-  // (requires that the ElectrodeGroup has been written)
-  if (!electrodeTableCreated) {
-    m_electrodeTable->finalize();
-  }
-
   return Status::Success;
 }
 
@@ -341,7 +345,7 @@ Status NWBFile::createAnnotationSeries(std::vector<std::string> recordingNames,
     const std::string& recordingName = recordingNames[i];
 
     std::string annotationSeriesPath =
-        AQNWB::mergePaths(acquisitionPath, recordingName);
+        AQNWB::mergePaths(m_acquisitionPath, recordingName);
 
     // Setup annotation series datasets
     auto annotationSeries =

src/nwb/NWBFile.hpp

@@ -92,6 +92,20 @@ class NWBFile : public Container
    */
   Status finalize();
 
+  /**
+   * @brief Create ElectrodesTable.
+   * Note, this function will fail if the file is in a mode where
+   * new objects cannot be added, which can be checked via
+   * nwbfile.io->canModifyObjects()
+   * @param recordingArrays vector of ChannelVector indicating the electrodes to
+   *                        add to the table. This vector should contain all the
+   *                        electrodes that are detected by the acquisition
+   * system, not only those being actively recorded from.
+   * @return Status The status of the object creation operation.
+   */
+  Status createElectrodesTable(
+      std::vector<Types::ChannelVector> recordingArrays);
+
   /**
    * @brief Create ElectricalSeries objects to record data into.
    * Created objects are stored in recordingContainers.
@@ -112,9 +126,9 @@ class NWBFile : public Container
   Status createElectricalSeries(
       std::vector<Types::ChannelVector> recordingArrays,
       std::vector<std::string> recordingNames,
-      const IO::BaseDataType& dataType = IO::BaseDataType::I16,
-      RecordingContainers* recordingContainers = nullptr,
-      std::vector<SizeType>& containerIndexes = emptyContainerIndexes);
+      const IO::BaseDataType& dataType,
+      RecordingContainers* recordingContainers,
+      std::vector<SizeType>& containerIndexes);
 
   /**
    * @brief Create SpikeEventSeries objects to record data into.
@@ -133,9 +147,9 @@ class NWBFile : public Container
   Status createSpikeEventSeries(
       std::vector<Types::ChannelVector> recordingArrays,
       std::vector<std::string> recordingNames,
-      const IO::BaseDataType& dataType = IO::BaseDataType::I16,
-      RecordingContainers* recordingContainers = nullptr,
-      std::vector<SizeType>& containerIndexes = emptyContainerIndexes);
+      const IO::BaseDataType& dataType,
+      RecordingContainers* recordingContainers,
+      std::vector<SizeType>& containerIndexes);
 
   /** @brief Create AnnotationSeries objects to record data into.
    * Created objects are stored in recordingContainers.
@@ -146,10 +160,9 @@ class NWBFile : public Container
    * recordingContainers
    * @return Status The status of the object creation operation.
    */
-  Status createAnnotationSeries(
-      std::vector<std::string> recordingNames,
-      RecordingContainers* recordingContainers = nullptr,
-      std::vector<SizeType>& containerIndexes = emptyContainerIndexes);
+  Status createAnnotationSeries(std::vector<std::string> recordingNames,
+                                RecordingContainers* recordingContainers,
+                                std::vector<SizeType>& containerIndexes);
 
   DEFINE_REGISTERED_FIELD(readElectrodeTable,
                           ElectrodeTable,
@@ -246,10 +259,8 @@ class NWBFile : public Container
       const std::array<std::pair<std::string_view, std::string_view>, N>&
           specVariables);
 
-  inline const static std::string acquisitionPath = "/acquisition";
-  static std::vector<SizeType> emptyContainerIndexes;
+  inline const static std::string m_acquisitionPath = "/acquisition";
 
-private:
   /**
    * @brief The ElectrodeTable for the file
    */