calibration: add voltage_data to item

changelog.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## v1.7.0 | t.b.d.
+
+* Added [voltage](https://lumicks-pylake.readthedocs.io/en/latest/_api/lumicks.pylake.calibration.ForceCalibrationItem.html#lumicks.pylake.calibration.ForceCalibrationItem.voltage), [sum_voltage](https://lumicks-pylake.readthedocs.io/en/latest/_api/lumicks.pylake.calibration.ForceCalibrationItem.html#lumicks.pylake.calibration.ForceCalibrationItem.sum_voltage) and [driving](https://lumicks-pylake.readthedocs.io/en/latest/_api/lumicks.pylake.calibration.ForceCalibrationItem.html#lumicks.pylake.calibration.ForceCalibrationItem.driving) properties which return raw calibration data. Note that these are only available when the option to export the raw data has explicitly been selected in Bluelake. When unavailable, these properties will return empty slices.
+
 ## v1.6.0 | t.b.d.
 
 #### New features

docs/tutorial/force_calibration/calibration_items.rst

@@ -84,6 +84,12 @@ To do this, we divide our data by the force sensitivity that was active at the s
     >>> old_calibration = force1x_slice.calibration[0]
     ... volts1x_slice = force1x_slice / old_calibration.force_sensitivity
 
+.. note::
+
+    If you exported the raw data with the calibration items from Bluelake you can skip the slicing
+    and decalibration steps and use these raw calibration voltages directly by invoking
+    `volts1x_slice = f.force1x.calibration[1].voltage`.
+
 The easiest way to extract all the relevant input parameters for a calibration is to use
 :meth:`~lumicks.pylake.calibration.calibration_item.ForceCalibrationItem.calibration_params()`::
 

lumicks/pylake/calibration.py

@@ -2,6 +2,7 @@
 
 from tabulate import tabulate
 
+from lumicks.pylake.channel import Slice, Continuous
 from lumicks.pylake.force_calibration.calibration_item import ForceCalibrationItem
 
 
@@ -39,7 +40,7 @@ class ForceCalibrationList:
     """
 
     def __init__(self, items, slice_start=None, slice_stop=None):
-        """Calibration item
+        """List of calibration items
 
         Parameters
         ----------
@@ -100,17 +101,48 @@ def from_field(hdf5, force_channel) -> "ForceCalibrationList":
             Calibration field to access (e.g. "Force 1x").
         """
 
+        def make_slice(dset, field, y_label, title) -> Slice:
+            """Fetch raw data from the dataset"""
+            if field in dset:
+                return Slice(
+                    Continuous.from_dataset(dset[field]),
+                    labels={"x": "Time (s)", "y": y_label, "title": title},
+                )
+
         if "Calibration" not in hdf5.keys():
             return ForceCalibrationList(items=[])
 
         items = []
         for calibration_item in hdf5["Calibration"].values():
             if force_channel in calibration_item:
-                attrs = dict(calibration_item[force_channel].attrs)
+                dset = calibration_item[force_channel]
+                attrs = dict(dset.attrs)
 
                 # Copy the timestamp at which the calibration was applied into the item
                 attrs["Timestamp (ns)"] = calibration_item.attrs.get("Timestamp (ns)")
-                items.append(ForceCalibrationItem(attrs))
+                items.append(
+                    ForceCalibrationItem(
+                        attrs,
+                        voltage=make_slice(
+                            dset,
+                            "voltage",
+                            "Uncalibrated Force (V)",
+                            f"Uncalibrated {force_channel}",
+                        ),
+                        sum_voltage=make_slice(
+                            dset,
+                            "sum_voltage",
+                            "Sum voltage (V)",
+                            f"Sum voltage {force_channel[-2]}",
+                        ),
+                        driving=make_slice(
+                            dset,
+                            "driving",
+                            r"Driving data ($\mu$m)",
+                            f"Driving data for axis {force_channel[-1]}",
+                        ),
+                    )
+                )
 
         return ForceCalibrationList._from_items(items)
 
@@ -137,7 +169,8 @@ def format_timestamp(timestamp):
                     ),
                     item.hydrodynamically_correct,
                     item.distance_to_surface is not None,
-                    (
+                    item.has_data  # Data in the item itself
+                    or (  # Data on the slice
                         bool(
                             self._slice_start
                             and (item.start >= self._slice_start)

lumicks/pylake/channel.py

@@ -208,7 +208,7 @@ def stop(self):
         return self._src.stop
 
     @property
-    def data(self) -> npt.ArrayLike:
+    def data(self) -> npt.NDArray:
         """The primary values of this channel slice"""
         return self._src.data
 
@@ -801,7 +801,7 @@ def to_dataset(self, parent, name, **kwargs):
         return dset
 
     @property
-    def data(self) -> npt.ArrayLike:
+    def data(self) -> npt.NDArray:
         if self._cached_data is None:
             self._cached_data = np.asarray(self._src_data)
         return self._cached_data
@@ -932,7 +932,7 @@ def to_dataset(self, parent, name, **kwargs):
         return dset
 
     @property
-    def data(self) -> npt.ArrayLike:
+    def data(self) -> npt.NDArray:
         if self._cached_data is None:
             self._cached_data = np.asarray(self._src_data)
         return self._cached_data
@@ -1047,7 +1047,7 @@ def __len__(self):
         return 0
 
     @property
-    def data(self) -> npt.ArrayLike:
+    def data(self) -> npt.NDArray:
         return np.empty(0)
 
     @property

lumicks/pylake/force_calibration/calibration_item.py

@@ -2,13 +2,43 @@
 from functools import wraps
 from collections import UserDict
 
+from lumicks.pylake.channel import empty_slice
+from lumicks.pylake.force_calibration.convenience import calibrate_force
 from lumicks.pylake.force_calibration.calibration_models import DiodeCalibrationModel
 from lumicks.pylake.force_calibration.detail.calibration_properties import (
     CalibrationPropertiesMixin,
 )
 
 
 class ForceCalibrationItem(UserDict, CalibrationPropertiesMixin):
+    def __init__(
+        self,
+        dictionary=None,
+        *,
+        voltage=None,
+        sum_voltage=None,
+        driving=None,
+    ):
+        super().__init__(dictionary)
+        self._voltage = voltage
+        self._sum_voltage = sum_voltage
+        self._driving = driving
+
+    @property
+    def voltage(self):
+        """Uncalibrated voltage reading on the detector"""
+        return self._voltage if self._voltage else empty_slice
+
+    @property
+    def sum_voltage(self):
+        """Uncalibrated sum voltage on the detector"""
+        return self._sum_voltage if self._sum_voltage else empty_slice
+
+    @property
+    def driving(self):
+        """Driving signal used for active calibration"""
+        return self._driving if self._driving else empty_slice
+
     @staticmethod
     def _verify_full(method):
         @wraps(method)
@@ -129,6 +159,26 @@ def __repr__(self):
         )
         return f"{self.__class__.__name__}({properties})"
 
+    def plot(self):
+        if not self.voltage:
+            raise ValueError(
+                "This calibration item does not contain the raw data. If you still have the "
+                "timeline force data, you can de-calibrate that and perform the re-calibration"
+                "manually. See the pylake tutorial on force calibration for more information."
+            )
+
+        self.recalibrate_with().plot()
+
+    def recalibrate_with(self, **params):
+        """Returns a calibration structure with some parameters overridden.
+
+        For a full list of parameters to override, please see
+        :func:`~lumicks.pylake.calibrate_force()`"""
+        active_data = {"driving_data": self.driving.data} if self.active_calibration else {}
+        return calibrate_force(
+            self.voltage.data, **(self.calibration_params() | active_data | params)
+        )
+
     @_verify_full
     def _model_params(self):
         """Returns parameters with which to create an active or passive calibration model"""

lumicks/pylake/force_calibration/detail/calibration_properties.py

@@ -5,6 +5,11 @@
 
 
 class CalibrationPropertiesMixin:
+    @property
+    def has_data(self):
+        """Returns whether the calibration item has the raw data stored inside the item"""
+        return hasattr(self, "_voltage") and self._voltage is not None
+
     def _get_parameter(self, pylake_key, bluelake_key):
         raise NotImplementedError
 

lumicks/pylake/tests/data/mock_file.py

@@ -53,7 +53,9 @@ class MockDataFile_v2(MockDataFile_v1):
     def get_file_format_version(self):
         return 2
 
-    def make_calibration_data(self, calibration_idx, group, attributes, application_timestamp=None):
+    def make_calibration_data(
+        self, calibration_idx, group, attributes, application_timestamp=None, channels=None
+    ):
         if "Calibration" not in self.file:
             self.file.create_group("Calibration")
 
@@ -73,14 +75,27 @@ def make_calibration_data(self, calibration_idx, group, attributes, application_
         for i, v in attributes.items():
             field.attrs[i] = v
 
-    def make_fd(self, fd_name=None, metadata={}, attributes={}):
+        if channels:
+            for name, data in channels.items():
+                self.make_continuous_channel(
+                    group=f"Calibration/{calibration_idx}/{group}",
+                    name=name,
+                    start=attributes["Start time (ns)"],
+                    dt=np.int64(1e9 / 78125),
+                    data=data,
+                )
+
+    def make_fd(self, fd_name=None, metadata=None, attributes=None):
         if "FD Curve" not in self.file:
             self.file.create_group("FD Curve")
 
         if fd_name:
-            dset = self.file["FD Curve"].create_dataset(fd_name, data=metadata)
-            for i, v in attributes.items():
-                dset.attrs[i] = v
+            dset = self.file["FD Curve"].create_dataset(
+                fd_name, data={} if metadata is None else metadata
+            )
+            if attributes:
+                for i, v in attributes.items():
+                    dset.attrs[i] = v
 
     def make_marker(self, marker_name, attributes, payload=None):
         if "Marker" not in self.file:

lumicks/pylake/tests/test_file/conftest.py

@@ -67,7 +67,16 @@ def generate_attributes(stop_time, custom_fields=None):
         mock_file.make_calibration_data("1", "Force 1y", generate_attributes(0))
         mock_file.make_calibration_data("2", "Force 1y", generate_attributes(1))
         mock_file.make_calibration_data("3", "Force 1y", reset_attrs)
-        mock_file.make_calibration_data("3b", "Force 1y", generate_attributes(8))
+        mock_file.make_calibration_data(
+            "3b",
+            "Force 1y",
+            generate_attributes(8),
+            channels={
+                "sum_voltage": np.arange(5.0),
+                "voltage": np.arange(15.0),
+                "driving": np.arange(25.0),
+            },
+        )
         mock_file.make_calibration_data("4", "Force 1y", generate_attributes(10))
         mock_file.make_calibration_data("5", "Force 1y", generate_attributes(100))
         mock_file.make_calibration_data("1", "Force 1z", generate_attributes(0))

lumicks/pylake/tests/test_file/test_file_items.py

@@ -62,6 +62,31 @@ def test_calibration(h5_file):
         assert f.force1y.calibration[2].applied_at == 8
         assert f.force1y.calibration[3].applied_at == 10
         assert f.force2x.calibration[0].applied_at == 100
+        assert not f.force1x.calibration[0].has_data
+        assert len(f.force1x.calibration[0].voltage.data) == 0
+        assert len(f.force1x.calibration[0].sum_voltage.data) == 0
+        assert len(f.force1x.calibration[0].driving.data) == 0
+
+        # Verify that one specific calibration has data associated with it
+        idx = 2
+        assert not f.force1x.calibration[idx].has_data
+        np.testing.assert_equal(f.force1y.calibration[idx].sum_voltage.data, np.arange(5.0))
+        assert f.force1y.calibration[idx].sum_voltage.sample_rate == 78125
+        assert f.force1y.calibration[idx].sum_voltage.labels["x"] == "Time (s)"
+        assert f.force1y.calibration[idx].sum_voltage.labels["y"] == "Sum voltage (V)"
+        assert f.force1y.calibration[idx].sum_voltage.labels["title"] == "Sum voltage 1"
+
+        np.testing.assert_equal(f.force1y.calibration[idx].voltage.data, np.arange(15.0))
+        assert f.force1y.calibration[idx].voltage.sample_rate == 78125
+        assert f.force1y.calibration[idx].voltage.labels["x"] == "Time (s)"
+        assert f.force1y.calibration[idx].voltage.labels["y"] == "Uncalibrated Force (V)"
+        assert f.force1y.calibration[idx].voltage.labels["title"] == "Uncalibrated Force 1y"
+
+        np.testing.assert_equal(f.force1y.calibration[idx].driving.data, np.arange(25.0))
+        assert f.force1y.calibration[idx].driving.sample_rate == 78125
+        assert f.force1y.calibration[idx].driving.labels["x"] == "Time (s)"
+        assert f.force1y.calibration[idx].driving.labels["y"] == r"Driving data ($\mu$m)"
+        assert f.force1y.calibration[idx].driving.labels["title"] == "Driving data for axis y"
 
 
 def test_marker(h5_file):