From 31ac7b64eca5d449159a7bb53dd18918be0a977e Mon Sep 17 00:00:00 2001 From: Jvshen Date: Thu, 9 Jan 2025 14:41:55 -0800 Subject: [PATCH 01/38] Add attribute for 3d triangulation --- sleap/gui/commands.py | 2 +- sleap/io/cameras.py | 79 +++++++++++++++++++++++++++++++++++-------- 2 files changed, 66 insertions(+), 15 deletions(-) diff --git a/sleap/gui/commands.py b/sleap/gui/commands.py index 075ccf060..03f183a49 100644 --- a/sleap/gui/commands.py +++ b/sleap/gui/commands.py @@ -3785,7 +3785,7 @@ def do_action(cls, context: CommandContext, params: dict): # Update or create/insert ("upsert") instance points frame_group.upsert_points( - points=points_reprojected, + points_3d=points_reprojected, instance_groups=instance_groups, exclude_complete=True, ) diff --git a/sleap/io/cameras.py b/sleap/io/cameras.py index c2a1bb831..3191ef1a8 100644 --- a/sleap/io/cameras.py +++ b/sleap/io/cameras.py @@ -16,6 +16,7 @@ from sleap.instance import Instance, LabeledFrame, PredictedInstance from sleap.io.video import Video from sleap.util import compute_oks, deep_iterable_converter +from sleap_anipose.triangulation import reproject logger = logging.getLogger(__name__) @@ -452,6 +453,7 @@ class InstanceGroup: _dummy_instance: Optional[Instance] = field(default=None) camera_cluster: Optional[CameraCluster] = field(default=None) _score: Optional[float] = field(default=None) + _triangulation: Optional[np.ndarray] = field(default=None) def __attrs_post_init__(self): """Initialize `InstanceGroup` object.""" @@ -460,6 +462,14 @@ def __attrs_post_init__(self): for cam, instance in self._instance_by_camcorder.items(): self._camcorder_by_instance[instance] = cam + @property + def triangulation(self) -> Optional[np.ndarray]: + return self._triangulation + + @triangulation.setter + def triangulation(self, triangulation: np.ndarray): + self._triangulation = triangulation + def _create_dummy_instance(self, instance: Optional[Instance] = None): """Create a dummy instance to fill in for missing instances. @@ -837,7 +847,8 @@ def get_cam(self, instance: Instance) -> Optional[Camcorder]: def update_points( self, - points: np.ndarray, + points_3d: np.ndarray, + instance_groups: List['InstanceGroup'], cams_to_include: Optional[List[Camcorder]] = None, exclude_complete: bool = True, ): @@ -853,13 +864,62 @@ def update_points( exclude_complete: If True, then do not update points that are marked as complete. Default is True. """ + # Ensure we are working with a float array + points_3d = points_3d.astype(float) + + # Check if points are 3D + is_3d = points_3d.shape[-1] == 3 + if not is_3d: + raise ValueError("Expected 3D points with shape (M, T, N, 3).") + + # Check that the correct shape was passed in + n_views, n_instances, n_nodes, n_coords = points_3d.shape + assert n_views == len( + self.cams_to_include + ), f"Expected {len(self.cams_to_include)} views, got {n_views}." + assert n_instances == len( + instance_groups + ), f"Expected {len(instance_groups)} instances, got {n_instances}." + assert n_coords == 3, f"Expected 3 coordinates, got {n_coords}." + + # Reproject 3D points into 2D points for each camera view + pts_reprojected = reproject( + points_3d, + calib=self.session.camera_cluster, + excluded_views=self.excluded_views, + ) # M=include x F=1 x T x N x 2 + + # Squeeze back to the original shape + points_reprojected = np.squeeze(pts_reprojected, axis=1) # M=include x TxNx2 + + # Get projection bounds (based on video height/width) + bounds = self.session.projection_bounds + bounds_expanded_x = bounds[:, None, None, 0] + bounds_expanded_y = bounds[:, None, None, 1] + + # Create masks for out-of-bounds x and y coordinates + out_of_bounds_x = (points_reprojected[..., 0] < 0) | (points_reprojected[..., 0] > bounds_expanded_x) + out_of_bounds_y = (points_reprojected[..., 1] < 0) | (points_reprojected[..., 1] > bounds_expanded_y) + + # Replace out-of-bounds x and y coordinates with nan + points_reprojected[out_of_bounds_x, 0] = np.nan + points_reprojected[out_of_bounds_y, 1] = np.nan + + # Update points for each `InstanceGroup` + for ig_idx, instance_group in enumerate(instance_groups): + # Ensure that `InstanceGroup`s is in this `FrameGroup` + self._raise_if_instance_group_not_in_frame_group( + instance_group=instance_group + ) + # Update points for the instance group + instance_group.points = points_reprojected[ig_idx] # If no `Camcorder`s specified, then update `Instance`s for all `CameraCluster` if cams_to_include is None: cams_to_include = self.camera_cluster.cameras # Check that correct shape was passed in - n_views, n_nodes, _ = points.shape + n_views, n_nodes, _ = points_3d.shape assert n_views == len(cams_to_include), ( f"Number of views in `points` ({n_views}) does not match the number of " f"Camcorders in `cams_to_include` ({len(cams_to_include)})." @@ -883,13 +943,13 @@ def update_points( if not isinstance(instance, PredictedInstance): instance_oks = compute_oks( gt_points[cam_idx, :, :], - points[cam_idx, :, :], + points_3d[cam_idx, :, :], ) oks_scores[cam_idx] = instance_oks # Update the points for the instance instance.update_points( - points=points[cam_idx, :, :], exclude_complete=exclude_complete + points=points_3d[cam_idx, :, :], exclude_complete=exclude_complete ) # Update the score for the InstanceGroup to be the average OKS score @@ -2289,16 +2349,6 @@ def upsert_points( complete. Default is True. """ - # Check that the correct shape was passed in - n_views, n_instances, n_nodes, n_coords = points.shape - assert n_views == len( - self.cams_to_include - ), f"Expected {len(self.cams_to_include)} views, got {n_views}." - assert n_instances == len( - instance_groups - ), f"Expected {len(instance_groups)} instances, got {n_instances}." - assert n_coords == 2, f"Expected 2 coordinates, got {n_coords}." - # Ensure we are working with a float array points = points.astype(float) @@ -2331,6 +2381,7 @@ def upsert_points( points=instance_points, cams_to_include=self.cams_to_include, exclude_complete=exclude_complete, + bounds=bounds, ) def _raise_if_instance_not_in_instance_group(self, instance: Instance): From ba0420959e6b7f397978bcc2f60918179bb1cadd Mon Sep 17 00:00:00 2001 From: Jvshen Date: Mon, 13 Jan 2025 16:59:37 -0800 Subject: [PATCH 02/38] Added projection bounds and excluded views inputs to update_points and updated doc string --- sleap/io/cameras.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/sleap/io/cameras.py b/sleap/io/cameras.py index 3191ef1a8..995e89336 100644 --- a/sleap/io/cameras.py +++ b/sleap/io/cameras.py @@ -847,22 +847,25 @@ def get_cam(self, instance: Instance) -> Optional[Camcorder]: def update_points( self, - points_3d: np.ndarray, - instance_groups: List['InstanceGroup'], + points: np.ndarray, + projection_bounds: np.ndarray, cams_to_include: Optional[List[Camcorder]] = None, exclude_complete: bool = True, + excluded_views: Optional[List[str]] = None, ): """Update the points in the `Instance` for the specified `Camcorder`s. Args: - points: Numpy array of shape (M, N, 2) where M is the number of views, N is - the number of Nodes, and 2 is for x, y. + points: Numpy array of shape (N, 3) N is + the number of Nodes, and 3 is for x, y, z. + projections_bounds: Numpy array of shape (M, 2) where M is the number of views cams_to_include: List of `Camcorder`s to include in the update. The order of the `Camcorder`s in the list should match the order of the views in the `points` array. If None, then all `Camcorder`s in the `CameraCluster` are included. Default is None. exclude_complete: If True, then do not update points that are marked as complete. Default is True. + excluded_views: List of `Camcorder` names to exclude from the update. """ # Ensure we are working with a float array points_3d = points_3d.astype(float) From a32dbaab8406a756e09558bdb4141e87d2f5278d Mon Sep 17 00:00:00 2001 From: Jvshen Date: Mon, 13 Jan 2025 17:03:14 -0800 Subject: [PATCH 03/38] Verify inputs n_coords, cams_to_include, excluded_views, len(self.camera_cluster) --- sleap/io/cameras.py | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/sleap/io/cameras.py b/sleap/io/cameras.py index 995e89336..e5884acc2 100644 --- a/sleap/io/cameras.py +++ b/sleap/io/cameras.py @@ -868,22 +868,29 @@ def update_points( excluded_views: List of `Camcorder` names to exclude from the update. """ # Ensure we are working with a float array - points_3d = points_3d.astype(float) + points = points.astype(float) - # Check if points are 3D - is_3d = points_3d.shape[-1] == 3 - if not is_3d: - raise ValueError("Expected 3D points with shape (M, T, N, 3).") - # Check that the correct shape was passed in - n_views, n_instances, n_nodes, n_coords = points_3d.shape - assert n_views == len( - self.cams_to_include - ), f"Expected {len(self.cams_to_include)} views, got {n_views}." - assert n_instances == len( - instance_groups - ), f"Expected {len(instance_groups)} instances, got {n_instances}." - assert n_coords == 3, f"Expected 3 coordinates, got {n_coords}." + points = points.squeeze() # N x 3 + n_nodes, n_coords = points.shape + if n_coords != 3: + raise ValueError( + f"Expected 3 coordinates in `points`, got {n_coords}." + ) + + # If no `Camcorder`s specified, then update `Instance`s for all `CameraCluster` + if cams_to_include is None: + cams_to_include = self.camera_cluster.cameras + + if excluded_views is None: + excluded_views = () + + if len(cams_to_include) + len(excluded_views) != len(self.camera_cluster): + raise ValueError( + f"The number of `Camcorder`s to include {len(cams_to_include)} plus the number of `Camcorder`s " + f"to exclude {len(excluded_views)} does not match the number of `Camcorder`s in the " + f"`CameraCluster` {len(self.camera_cluster)}." + ) # Reproject 3D points into 2D points for each camera view pts_reprojected = reproject( From 3715a59d8d1994095bfeb08bb2dd36aeda33af20 Mon Sep 17 00:00:00 2001 From: Jvshen Date: Mon, 13 Jan 2025 17:04:01 -0800 Subject: [PATCH 04/38] Fix shapes --- sleap/io/cameras.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/sleap/io/cameras.py b/sleap/io/cameras.py index e5884acc2..0f77212c7 100644 --- a/sleap/io/cameras.py +++ b/sleap/io/cameras.py @@ -894,18 +894,18 @@ def update_points( # Reproject 3D points into 2D points for each camera view pts_reprojected = reproject( - points_3d, - calib=self.session.camera_cluster, - excluded_views=self.excluded_views, + np.expand_dims(points, axis=(0, 1)), # M=include x N x 3 + calib=self.camera_cluster, + excluded_views=excluded_views, ) # M=include x F=1 x T x N x 2 # Squeeze back to the original shape - points_reprojected = np.squeeze(pts_reprojected, axis=1) # M=include x TxNx2 + points_reprojected = np.squeeze(pts_reprojected, axis=(1, 2)) # M=include x Nx2 # Get projection bounds (based on video height/width) - bounds = self.session.projection_bounds - bounds_expanded_x = bounds[:, None, None, 0] - bounds_expanded_y = bounds[:, None, None, 1] + bounds = projection_bounds #TODO: make sure projection bounds are the shape they need to be in update points + bounds_expanded_x = bounds[:, None, 0] + bounds_expanded_y = bounds[:, None, 1] # Create masks for out-of-bounds x and y coordinates out_of_bounds_x = (points_reprojected[..., 0] < 0) | (points_reprojected[..., 0] > bounds_expanded_x) From 68e5746791ccc795e5699a8cb6e9b12abc497a3e Mon Sep 17 00:00:00 2001 From: Jvshen Date: Mon, 13 Jan 2025 17:05:53 -0800 Subject: [PATCH 05/38] Rename points_3d to points_reprojected --- sleap/io/cameras.py | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/sleap/io/cameras.py b/sleap/io/cameras.py index 0f77212c7..57efdd741 100644 --- a/sleap/io/cameras.py +++ b/sleap/io/cameras.py @@ -915,21 +915,8 @@ def update_points( points_reprojected[out_of_bounds_x, 0] = np.nan points_reprojected[out_of_bounds_y, 1] = np.nan - # Update points for each `InstanceGroup` - for ig_idx, instance_group in enumerate(instance_groups): - # Ensure that `InstanceGroup`s is in this `FrameGroup` - self._raise_if_instance_group_not_in_frame_group( - instance_group=instance_group - ) - # Update points for the instance group - instance_group.points = points_reprojected[ig_idx] - - # If no `Camcorder`s specified, then update `Instance`s for all `CameraCluster` - if cams_to_include is None: - cams_to_include = self.camera_cluster.cameras - # Check that correct shape was passed in - n_views, n_nodes, _ = points_3d.shape + n_views, n_nodes, _ = points_reprojected.shape assert n_views == len(cams_to_include), ( f"Number of views in `points` ({n_views}) does not match the number of " f"Camcorders in `cams_to_include` ({len(cams_to_include)})." @@ -953,13 +940,13 @@ def update_points( if not isinstance(instance, PredictedInstance): instance_oks = compute_oks( gt_points[cam_idx, :, :], - points_3d[cam_idx, :, :], + points_reprojected[cam_idx, :, :], ) oks_scores[cam_idx] = instance_oks # Update the points for the instance instance.update_points( - points=points_3d[cam_idx, :, :], exclude_complete=exclude_complete + points=points_reprojected[cam_idx, :, :], exclude_complete=exclude_complete ) # Update the score for the InstanceGroup to be the average OKS score From f3bce0dadb1ed98799ff44e20d4c2561d0958444 Mon Sep 17 00:00:00 2001 From: Jvshen Date: Mon, 13 Jan 2025 17:08:08 -0800 Subject: [PATCH 06/38] Change tests to match changes --- tests/io/test_cameras.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/io/test_cameras.py b/tests/io/test_cameras.py index 97c2de94f..e606c1255 100644 --- a/tests/io/test_cameras.py +++ b/tests/io/test_cameras.py @@ -714,23 +714,26 @@ def test_instance_group( equal_nan=True, ) + projection_bounds = np.full((len(instance_group.camera_cluster), 2), np.nan) # Test `update_points` method assert not np.all(instance_group.numpy(invisible_as_nan=False) == 72317) # Remove some Instances to "expose" underlying PredictedInstances for inst in instance_group.instances[:2]: lf = inst.frame labels.remove_instance(lf, inst) - instance_group.update_points(points=np.full((n_views, n_nodes, n_coords), 72317)) + instance_group.update_points(points=np.full((n_nodes, 3), 72317), projection_bounds=projection_bounds) for inst in instance_group.instances: if isinstance(inst, PredictedInstance): assert inst.score == instance_group.score prev_score = instance_group.score - instance_group.update_points(points=np.full((n_views, n_nodes, n_coords), 72317)) + instance_group.update_points(points=np.full((n_nodes, 3), 72317), projection_bounds=projection_bounds) for inst in instance_group.instances: if isinstance(inst, PredictedInstance): assert inst.score == instance_group.score instance_group_numpy = instance_group.numpy(invisible_as_nan=False) - assert np.all(instance_group_numpy == 72317) + for inst_group_np in instance_group_numpy: + assert np.all(inst_group_np[:,0] == inst_group_np[0,0]) + assert np.all(inst_group_np[:,1] == inst_group_np[0,1]) assert instance_group.score == 1.0 # Score should be 1.0 because same points # Test `add_instance`, `replace_instance`, and `remove_instance` From 9ed8417561f357542699d44ecb525bfed1ac227f Mon Sep 17 00:00:00 2001 From: Jvshen Date: Wed, 15 Jan 2025 13:03:15 -0800 Subject: [PATCH 07/38] Set excluded_complete to False in tests --- tests/io/test_cameras.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/io/test_cameras.py b/tests/io/test_cameras.py index e606c1255..674c36f54 100644 --- a/tests/io/test_cameras.py +++ b/tests/io/test_cameras.py @@ -721,12 +721,12 @@ def test_instance_group( for inst in instance_group.instances[:2]: lf = inst.frame labels.remove_instance(lf, inst) - instance_group.update_points(points=np.full((n_nodes, 3), 72317), projection_bounds=projection_bounds) + instance_group.update_points(points=np.full((n_nodes, 3), 72317), projection_bounds=projection_bounds, exclude_complete=False) for inst in instance_group.instances: if isinstance(inst, PredictedInstance): assert inst.score == instance_group.score prev_score = instance_group.score - instance_group.update_points(points=np.full((n_nodes, 3), 72317), projection_bounds=projection_bounds) + instance_group.update_points(points=np.full((n_nodes, 3), 72317), projection_bounds=projection_bounds, exclude_complete=False) for inst in instance_group.instances: if isinstance(inst, PredictedInstance): assert inst.score == instance_group.score From b07e30527fe1eb3e6c23d464e85c93090187b295 Mon Sep 17 00:00:00 2001 From: Jvshen Date: Thu, 23 Jan 2025 11:08:23 -0800 Subject: [PATCH 08/38] Add inputs for update_points in upsert_points --- sleap/io/cameras.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sleap/io/cameras.py b/sleap/io/cameras.py index 57efdd741..7bb2969ef 100644 --- a/sleap/io/cameras.py +++ b/sleap/io/cameras.py @@ -2373,12 +2373,13 @@ def upsert_points( self.create_and_add_missing_instances(instance_group=instance_group) # Update points for each `Instance` in `InstanceGroup` - instance_points = points[:, ig_idx, :, :] # M x N x 2 + instance_points = points[ig_idx, :, :] # N x 3 instance_group.update_points( points=instance_points, cams_to_include=self.cams_to_include, + excluded_views=self.excluded_views, exclude_complete=exclude_complete, - bounds=bounds, + projection_bounds=bounds, ) def _raise_if_instance_not_in_instance_group(self, instance: Instance): From 49defef0bd620ea0f9e6ebb6548c35b63b648f03 Mon Sep 17 00:00:00 2001 From: Jvshen Date: Thu, 23 Jan 2025 11:10:07 -0800 Subject: [PATCH 09/38] Remove projection bounds logic in upsert_points --- sleap/io/cameras.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/sleap/io/cameras.py b/sleap/io/cameras.py index 7bb2969ef..767621c40 100644 --- a/sleap/io/cameras.py +++ b/sleap/io/cameras.py @@ -2351,16 +2351,6 @@ def upsert_points( # Get projection bounds (based on video height/width) bounds = self.session.projection_bounds - bounds_expanded_x = bounds[:, None, None, 0] - bounds_expanded_y = bounds[:, None, None, 1] - - # Create masks for out-of-bounds x and y coordinates - out_of_bounds_x = (points[..., 0] < 0) | (points[..., 0] > bounds_expanded_x) - out_of_bounds_y = (points[..., 1] < 0) | (points[..., 1] > bounds_expanded_y) - - # Replace out-of-bounds x and y coordinates with nan - points[out_of_bounds_x, 0] = np.nan - points[out_of_bounds_y, 1] = np.nan # Update points for each `InstanceGroup` for ig_idx, instance_group in enumerate(instance_groups): From c4db7b44131d6c592c2b0c9d85536852b52bc5f5 Mon Sep 17 00:00:00 2001 From: roomrys <38435167+roomrys@users.noreply.github.com> Date: Thu, 23 Jan 2025 11:34:48 -0800 Subject: [PATCH 10/38] Lint --- tests/io/test_cameras.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/tests/io/test_cameras.py b/tests/io/test_cameras.py index 674c36f54..30c2771d2 100644 --- a/tests/io/test_cameras.py +++ b/tests/io/test_cameras.py @@ -721,19 +721,27 @@ def test_instance_group( for inst in instance_group.instances[:2]: lf = inst.frame labels.remove_instance(lf, inst) - instance_group.update_points(points=np.full((n_nodes, 3), 72317), projection_bounds=projection_bounds, exclude_complete=False) + instance_group.update_points( + points=np.full((n_nodes, 3), 72317), + projection_bounds=projection_bounds, + exclude_complete=False, + ) for inst in instance_group.instances: if isinstance(inst, PredictedInstance): assert inst.score == instance_group.score prev_score = instance_group.score - instance_group.update_points(points=np.full((n_nodes, 3), 72317), projection_bounds=projection_bounds, exclude_complete=False) + instance_group.update_points( + points=np.full((n_nodes, 3), 72317), + projection_bounds=projection_bounds, + exclude_complete=False, + ) for inst in instance_group.instances: if isinstance(inst, PredictedInstance): assert inst.score == instance_group.score instance_group_numpy = instance_group.numpy(invisible_as_nan=False) for inst_group_np in instance_group_numpy: - assert np.all(inst_group_np[:,0] == inst_group_np[0,0]) - assert np.all(inst_group_np[:,1] == inst_group_np[0,1]) + assert np.all(inst_group_np[:, 0] == inst_group_np[0, 0]) + assert np.all(inst_group_np[:, 1] == inst_group_np[0, 1]) assert instance_group.score == 1.0 # Score should be 1.0 because same points # Test `add_instance`, `replace_instance`, and `remove_instance` From b0b9254512d913f876c4439e17c44fc563ac8008 Mon Sep 17 00:00:00 2001 From: roomrys <38435167+roomrys@users.noreply.github.com> Date: Thu, 23 Jan 2025 11:37:31 -0800 Subject: [PATCH 11/38] Add InstanceGroup.update_points_from_2d method --- sleap/io/cameras.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/sleap/io/cameras.py b/sleap/io/cameras.py index 767621c40..f216389e8 100644 --- a/sleap/io/cameras.py +++ b/sleap/io/cameras.py @@ -902,6 +902,29 @@ def update_points( # Squeeze back to the original shape points_reprojected = np.squeeze(pts_reprojected, axis=(1, 2)) # M=include x Nx2 + # Update the points for each `Instance` in the `InstanceGroup` using 2d points + self.update_points_from_2d( + points_reprojected=points_reprojected, + projection_bounds=projection_bounds, + cams_to_include=cams_to_include, + exclude_complete=exclude_complete, + ) + + def update_points_from_2d( + self, + points_reprojected: np.ndarray, + projection_bounds: np.ndarray, + cams_to_include: Optional[List[Camcorder]] = None, + exclude_complete: bool = True, + ): + + # Ensure we are working with a float array + points_reprojected = points_reprojected.astype(np.float64) + + # If no `Camcorder`s specified, then update `Instance`s for all `CameraCluster` + if cams_to_include is None: + cams_to_include = self.camera_cluster.cameras + # Get projection bounds (based on video height/width) bounds = projection_bounds #TODO: make sure projection bounds are the shape they need to be in update points bounds_expanded_x = bounds[:, None, 0] From a342bcb34cb7bc6d04f167c9686a1c2738c401fd Mon Sep 17 00:00:00 2001 From: roomrys <38435167+roomrys@users.noreply.github.com> Date: Thu, 23 Jan 2025 11:39:51 -0800 Subject: [PATCH 12/38] Lint --- sleap/io/cameras.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/sleap/io/cameras.py b/sleap/io/cameras.py index f216389e8..b5291f704 100644 --- a/sleap/io/cameras.py +++ b/sleap/io/cameras.py @@ -874,17 +874,15 @@ def update_points( points = points.squeeze() # N x 3 n_nodes, n_coords = points.shape if n_coords != 3: - raise ValueError( - f"Expected 3 coordinates in `points`, got {n_coords}." - ) - + raise ValueError(f"Expected 3 coordinates in `points`, got {n_coords}.") + # If no `Camcorder`s specified, then update `Instance`s for all `CameraCluster` if cams_to_include is None: cams_to_include = self.camera_cluster.cameras - + if excluded_views is None: excluded_views = () - + if len(cams_to_include) + len(excluded_views) != len(self.camera_cluster): raise ValueError( f"The number of `Camcorder`s to include {len(cams_to_include)} plus the number of `Camcorder`s " @@ -926,13 +924,17 @@ def update_points_from_2d( cams_to_include = self.camera_cluster.cameras # Get projection bounds (based on video height/width) - bounds = projection_bounds #TODO: make sure projection bounds are the shape they need to be in update points + bounds = projection_bounds # TODO: make sure projection bounds are the shape they need to be in update points bounds_expanded_x = bounds[:, None, 0] bounds_expanded_y = bounds[:, None, 1] # Create masks for out-of-bounds x and y coordinates - out_of_bounds_x = (points_reprojected[..., 0] < 0) | (points_reprojected[..., 0] > bounds_expanded_x) - out_of_bounds_y = (points_reprojected[..., 1] < 0) | (points_reprojected[..., 1] > bounds_expanded_y) + out_of_bounds_x = (points_reprojected[..., 0] < 0) | ( + points_reprojected[..., 0] > bounds_expanded_x + ) + out_of_bounds_y = (points_reprojected[..., 1] < 0) | ( + points_reprojected[..., 1] > bounds_expanded_y + ) # Replace out-of-bounds x and y coordinates with nan points_reprojected[out_of_bounds_x, 0] = np.nan @@ -969,7 +971,8 @@ def update_points_from_2d( # Update the points for the instance instance.update_points( - points=points_reprojected[cam_idx, :, :], exclude_complete=exclude_complete + points=points_reprojected[cam_idx, :, :], + exclude_complete=exclude_complete, ) # Update the score for the InstanceGroup to be the average OKS score From f84ddfb0ad062a7c88ca1c8a5fc7705d53469590 Mon Sep 17 00:00:00 2001 From: roomrys <38435167+roomrys@users.noreply.github.com> Date: Thu, 23 Jan 2025 11:40:49 -0800 Subject: [PATCH 13/38] Test InstanceGroup.update_points_from_2d --- tests/io/test_cameras.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/io/test_cameras.py b/tests/io/test_cameras.py index 30c2771d2..84e26b9f3 100644 --- a/tests/io/test_cameras.py +++ b/tests/io/test_cameras.py @@ -763,6 +763,34 @@ def test_instance_group( assert cam not in instance_group.cameras +def test_instance_group_update_points_from_2d( + multiview_min_session_frame_groups: Labels, +): + + labels = multiview_min_session_frame_groups + session: RecordingSession = labels.sessions[0] + frame_idx = 0 + frame_group = session.frame_groups[frame_idx] + instance_group = frame_group.instance_groups[0] + + # Test `update_points_from_2d` (all in bounds, all updated) + n_cameras = len(frame_group.cams_to_include) + n_instance_groups = 1 + n_nodes = len(frame_group.session.labels.skeleton.nodes) + n_coords = 2 + value = 100 + points = np.full((n_cameras, n_nodes, n_coords), value) + projection_bounds = frame_group.session.projection_bounds + cams_to_include = frame_group.cams_to_include + instance_group.update_points_from_2d( + points_reprojected=points, + projection_bounds=projection_bounds, + cams_to_include=cams_to_include, + exclude_complete=False, + ) + assert np.all(instance_group.numpy(invisible_as_nan=False) == value) + + def test_frame_group( multiview_min_session_labels: Labels, multiview_min_session_frame_groups: Labels ): From 5f5ef669f9a01f391bb006129f4ea6bca92624d8 Mon Sep 17 00:00:00 2001 From: roomrys <38435167+roomrys@users.noreply.github.com> Date: Thu, 23 Jan 2025 11:41:48 -0800 Subject: [PATCH 14/38] Fix input validation for update_points_from_2d --- sleap/io/cameras.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/sleap/io/cameras.py b/sleap/io/cameras.py index b5291f704..b41e87806 100644 --- a/sleap/io/cameras.py +++ b/sleap/io/cameras.py @@ -941,11 +941,14 @@ def update_points_from_2d( points_reprojected[out_of_bounds_y, 1] = np.nan # Check that correct shape was passed in - n_views, n_nodes, _ = points_reprojected.shape - assert n_views == len(cams_to_include), ( - f"Number of views in `points` ({n_views}) does not match the number of " - f"Camcorders in `cams_to_include` ({len(cams_to_include)})." - ) + n_views, n_nodes, n_coords = points_reprojected.shape + if n_views != len(cams_to_include): + raise ValueError( + f"Number of views in `points` ({n_views}) does not match the number of " + f"Camcorders in `cams_to_include` ({len(cams_to_include)})." + ) + if n_coords != 2: + raise ValueError(f"Expected 2 coordinates in `points`, got {n_coords}.") # Calculate OKS scores for the points gt_points = self.numpy( From 026c78f2ae0ffc3afacb38dd8911eb7126a5cb75 Mon Sep 17 00:00:00 2001 From: roomrys <38435167+roomrys@users.noreply.github.com> Date: Thu, 23 Jan 2025 11:48:37 -0800 Subject: [PATCH 15/38] Move shape check to top of function --- sleap/io/cameras.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/sleap/io/cameras.py b/sleap/io/cameras.py index b41e87806..449c495d8 100644 --- a/sleap/io/cameras.py +++ b/sleap/io/cameras.py @@ -916,6 +916,23 @@ def update_points_from_2d( exclude_complete: bool = True, ): + # Check that correct shape was passed in + points_shape = points_reprojected.shape + try: + n_views, n_nodes, n_coords = points_reprojected.shape + if n_views != len(cams_to_include): + raise ValueError( + f"Number of views in `points` ({n_views}) does not match the number" + f" of Camcorders in `cams_to_include` ({len(cams_to_include)})." + ) + if n_coords != 2: + raise ValueError(f"Expected 2 coordinates in `points`, got {n_coords}.") + except ValueError as e: + raise ValueError( + f"Expected `points_reprojected` to be of shape (M, N, 2), got " + f"{points_shape}.\n\n{e}" + ) + # Ensure we are working with a float array points_reprojected = points_reprojected.astype(np.float64) @@ -940,16 +957,6 @@ def update_points_from_2d( points_reprojected[out_of_bounds_x, 0] = np.nan points_reprojected[out_of_bounds_y, 1] = np.nan - # Check that correct shape was passed in - n_views, n_nodes, n_coords = points_reprojected.shape - if n_views != len(cams_to_include): - raise ValueError( - f"Number of views in `points` ({n_views}) does not match the number of " - f"Camcorders in `cams_to_include` ({len(cams_to_include)})." - ) - if n_coords != 2: - raise ValueError(f"Expected 2 coordinates in `points`, got {n_coords}.") - # Calculate OKS scores for the points gt_points = self.numpy( pred_as_nan=True, invisible_as_nan=True, cams_to_include=cams_to_include From 1e215396fab9c07afd3b3968588e020089ec9e4e Mon Sep 17 00:00:00 2001 From: roomrys <38435167+roomrys@users.noreply.github.com> Date: Thu, 23 Jan 2025 11:49:16 -0800 Subject: [PATCH 16/38] Test that all out of bounds means no update --- tests/io/test_cameras.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/io/test_cameras.py b/tests/io/test_cameras.py index 84e26b9f3..5aa1ad77e 100644 --- a/tests/io/test_cameras.py +++ b/tests/io/test_cameras.py @@ -790,6 +790,21 @@ def test_instance_group_update_points_from_2d( ) assert np.all(instance_group.numpy(invisible_as_nan=False) == value) + # Test `upsert_points` (all out of bound, none updated) + min_bound = projection_bounds.min() + prev_value = value + oob_value = 5000 + assert oob_value > min_bound + points = np.full((n_cameras, n_nodes, n_coords), oob_value) + instance_group.update_points_from_2d( + points_reprojected=points, + projection_bounds=projection_bounds, + cams_to_include=cams_to_include, + exclude_complete=False, + ) + assert np.any(instance_group.numpy(invisible_as_nan=False) == oob_value) == False + assert np.all(instance_group.numpy(invisible_as_nan=False) == prev_value) + def test_frame_group( multiview_min_session_labels: Labels, multiview_min_session_frame_groups: Labels From 36fd472a757bee03a99951c8ee6c373526aeaf71 Mon Sep 17 00:00:00 2001 From: roomrys <38435167+roomrys@users.noreply.github.com> Date: Thu, 23 Jan 2025 11:52:16 -0800 Subject: [PATCH 17/38] Test with some out of bound and some in bound --- tests/io/test_cameras.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/io/test_cameras.py b/tests/io/test_cameras.py index 5aa1ad77e..22a9211e9 100644 --- a/tests/io/test_cameras.py +++ b/tests/io/test_cameras.py @@ -805,6 +805,30 @@ def test_instance_group_update_points_from_2d( assert np.any(instance_group.numpy(invisible_as_nan=False) == oob_value) == False assert np.all(instance_group.numpy(invisible_as_nan=False) == prev_value) + # Test `upsert_points` (some out of bound, some updated) + value = 200 + oob_value = 5000 + assert oob_value > min_bound + oob_mask = np.random.choice([True, False], size=(n_cameras, n_nodes, n_coords)) + points = np.full((n_cameras, n_nodes, n_coords), value) + points[oob_mask] = oob_value + instance_group.update_points_from_2d( + points_reprojected=points, + projection_bounds=projection_bounds, + cams_to_include=cams_to_include, + exclude_complete=False, + ) + # Get the logical or for either x or y being out of bounds + oob_mask_1d = np.any(oob_mask, axis=-1) # Collapse last axis + oob_mask_1d_expanded = np.expand_dims(oob_mask_1d, axis=-1) + oob_mask_1d_expanded = np.broadcast_to(oob_mask_1d_expanded, oob_mask.shape) + instance_group_numpy = instance_group.numpy(invisible_as_nan=False) + assert np.any(instance_group_numpy > min_bound) == False + assert np.all( + instance_group_numpy[oob_mask_1d_expanded] == prev_value + ) # Not updated + assert np.all(instance_group_numpy[~oob_mask_1d_expanded] == value) # Updated + def test_frame_group( multiview_min_session_labels: Labels, multiview_min_session_frame_groups: Labels From ae2f317b30725f8029cbd670a2c1e3986c0f3d86 Mon Sep 17 00:00:00 2001 From: roomrys <38435167+roomrys@users.noreply.github.com> Date: Thu, 23 Jan 2025 11:56:56 -0800 Subject: [PATCH 18/38] Test some in and out of bounds with different bounds --- tests/io/test_cameras.py | 44 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/tests/io/test_cameras.py b/tests/io/test_cameras.py index 22a9211e9..6508b245b 100644 --- a/tests/io/test_cameras.py +++ b/tests/io/test_cameras.py @@ -829,6 +829,50 @@ def test_instance_group_update_points_from_2d( ) # Not updated assert np.all(instance_group_numpy[~oob_mask_1d_expanded] == value) # Updated + # Test `upsert_points` (between x,y bounds, some out of bound, some updated) + value = 300 + points = np.full((n_cameras, n_nodes, n_coords), value) + # Reset the points to all in bounds + instance_group.update_points_from_2d( + points_reprojected=points, + projection_bounds=projection_bounds, + cams_to_include=cams_to_include, + exclude_complete=False, + ) + assert np.all(instance_group.numpy(invisible_as_nan=False) == value) + # Add some out of bounds points + prev_value = value + value = 400 + points = np.full((n_cameras, n_nodes, n_coords), value) + max_bound = projection_bounds.max() + oob_value = max_bound - 1 + assert oob_value < max_bound and oob_value > min_bound + oob_mask = np.random.choice([True, False], size=(n_cameras, n_nodes, n_coords)) + points[oob_mask] = oob_value + instance_group.update_points_from_2d( + points_reprojected=points, + projection_bounds=projection_bounds, + cams_to_include=cams_to_include, + exclude_complete=False, + ) + # Get the logical or for either x or y being out of bounds + bound_x, bound_y = projection_bounds[:, 0].min(), projection_bounds[:, 1].min() + oob_mask_x = np.where(points[:, :, 0] > bound_x, True, False) + oob_mask_y = np.where(points[:, :, 1] > bound_y, True, False) + oob_mask_1d = np.logical_or(oob_mask_x, oob_mask_y) + oob_mask_1d_expanded = np.expand_dims(oob_mask_1d, axis=-1) + oob_mask_1d_expanded = np.broadcast_to(oob_mask_1d_expanded, oob_mask.shape) + instance_group_numpy = instance_group.numpy(invisible_as_nan=False) + assert np.any(instance_group_numpy[:, :, 0] > bound_x) == False + assert np.any(instance_group_numpy[:, :, 1] > bound_y) == False + assert np.all( + instance_group_numpy[oob_mask_1d_expanded] == prev_value + ) # Not updated + oob_value_mask = np.logical_and(~oob_mask_1d_expanded, oob_mask) + value_mask = np.logical_and(~oob_mask_1d_expanded, ~oob_mask) + assert np.all(instance_group_numpy[oob_value_mask] == oob_value) # Updated to oob + assert np.all(instance_group_numpy[value_mask] == value) # Updated to value + def test_frame_group( multiview_min_session_labels: Labels, multiview_min_session_frame_groups: Labels From 37bca05a0b334f34873423b19beef848d9de954d Mon Sep 17 00:00:00 2001 From: roomrys <38435167+roomrys@users.noreply.github.com> Date: Thu, 23 Jan 2025 11:57:16 -0800 Subject: [PATCH 19/38] Remove old test code --- tests/io/test_cameras.py | 84 ---------------------------------------- 1 file changed, 84 deletions(-) diff --git a/tests/io/test_cameras.py b/tests/io/test_cameras.py index 6508b245b..fd85c552e 100644 --- a/tests/io/test_cameras.py +++ b/tests/io/test_cameras.py @@ -1086,90 +1086,6 @@ def test_frame_group( assert labeled_frame_created in frame_group.labeled_frames assert labeled_frame in frame_group.session.labels.labeled_frames - # Test `upsert_points` (all in bounds, all updated) - n_cameras = len(frame_group.cams_to_include) - n_instance_groups = len(frame_group.instance_groups) - n_nodes = len(frame_group.session.labels.skeleton.nodes) - n_coords = 2 - value = 100 - points = np.full((n_cameras, n_instance_groups, n_nodes, n_coords), value) - frame_group.upsert_points( - points=points, instance_groups=frame_group.instance_groups - ) - assert np.all(frame_group.numpy(invisible_as_nan=False) == value) - - # Test `upsert_points` (all out of bound, none updated) - projection_bounds = frame_group.session.projection_bounds - min_bound = projection_bounds.min() - prev_value = value - oob_value = 5000 - assert oob_value > min_bound - points = np.full((n_cameras, n_instance_groups, n_nodes, n_coords), oob_value) - frame_group.upsert_points( - points=points, instance_groups=frame_group.instance_groups - ) - assert np.any(frame_group.numpy(invisible_as_nan=False) == oob_value) == False - assert np.all(frame_group.numpy(invisible_as_nan=False) == prev_value) - - # Test `upsert_points` (some out of bound, some updated) - value = 200 - oob_value = 5000 - assert oob_value > min_bound - oob_mask = np.random.choice( - [True, False], size=(n_cameras, n_instance_groups, n_nodes, n_coords) - ) - points = np.full((n_cameras, n_instance_groups, n_nodes, n_coords), value) - points[oob_mask] = oob_value - frame_group.upsert_points( - points=points, instance_groups=frame_group.instance_groups - ) - # Get the logical or for either x or y being out of bounds - oob_mask_1d = np.any(oob_mask, axis=-1) # Collapse last axis - oob_mask_1d_expanded = np.expand_dims(oob_mask_1d, axis=-1) - oob_mask_1d_expanded = np.broadcast_to(oob_mask_1d_expanded, oob_mask.shape) - frame_group_numpy = frame_group.numpy(invisible_as_nan=False) - assert np.any(frame_group_numpy > min_bound) == False - assert np.all(frame_group_numpy[oob_mask_1d_expanded] == prev_value) # Not updated - assert np.all(frame_group_numpy[~oob_mask_1d_expanded] == value) # Updated - - # Test `upsert_points` (between x,y bounds, some out of bound, some updated) - value = 300 - points = np.full((n_cameras, n_instance_groups, n_nodes, n_coords), value) - # Reset the points to all in bounds - frame_group.upsert_points( - points=points, instance_groups=frame_group.instance_groups - ) - assert np.all(frame_group.numpy(invisible_as_nan=False) == value) - # Add some out of bounds points - prev_value = value - value = 400 - points = np.full((n_cameras, n_instance_groups, n_nodes, n_coords), value) - max_bound = projection_bounds.max() - oob_value = max_bound - 1 - assert oob_value < max_bound and oob_value > min_bound - oob_mask = np.random.choice( - [True, False], size=(n_cameras, n_instance_groups, n_nodes, n_coords) - ) - points[oob_mask] = oob_value - frame_group.upsert_points( - points=points, instance_groups=frame_group.instance_groups - ) - # Get the logical or for either x or y being out of bounds - bound_x, bound_y = projection_bounds[:, 0].min(), projection_bounds[:, 1].min() - oob_mask_x = np.where(points[:, :, :, 0] > bound_x, True, False) - oob_mask_y = np.where(points[:, :, :, 1] > bound_y, True, False) - oob_mask_1d = np.logical_or(oob_mask_x, oob_mask_y) - oob_mask_1d_expanded = np.expand_dims(oob_mask_1d, axis=-1) - oob_mask_1d_expanded = np.broadcast_to(oob_mask_1d_expanded, oob_mask.shape) - frame_group_numpy = frame_group.numpy(invisible_as_nan=False) - assert np.any(frame_group_numpy[:, :, :, 0] > bound_x) == False - assert np.any(frame_group_numpy[:, :, :, 1] > bound_y) == False - assert np.all(frame_group_numpy[oob_mask_1d_expanded] == prev_value) # Not updated - oob_value_mask = np.logical_and(~oob_mask_1d_expanded, oob_mask) - value_mask = np.logical_and(~oob_mask_1d_expanded, ~oob_mask) - assert np.all(frame_group_numpy[oob_value_mask] == oob_value) # Updated to oob - assert np.all(frame_group_numpy[value_mask] == value) # Updated to value - def test_cameras_are_not_sorted(): """Test that cameras are not sorted in `RecordingSession`. From 64948f0bcf6d64b0f6a2ef63708ddd72bd0fff84 Mon Sep 17 00:00:00 2001 From: roomrys <38435167+roomrys@users.noreply.github.com> Date: Thu, 23 Jan 2025 11:57:51 -0800 Subject: [PATCH 20/38] Update comments in test --- tests/io/test_cameras.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/io/test_cameras.py b/tests/io/test_cameras.py index fd85c552e..f6be4fd31 100644 --- a/tests/io/test_cameras.py +++ b/tests/io/test_cameras.py @@ -790,7 +790,7 @@ def test_instance_group_update_points_from_2d( ) assert np.all(instance_group.numpy(invisible_as_nan=False) == value) - # Test `upsert_points` (all out of bound, none updated) + # Test `update_points_from_2d` (all out of bound, none updated) min_bound = projection_bounds.min() prev_value = value oob_value = 5000 @@ -805,7 +805,7 @@ def test_instance_group_update_points_from_2d( assert np.any(instance_group.numpy(invisible_as_nan=False) == oob_value) == False assert np.all(instance_group.numpy(invisible_as_nan=False) == prev_value) - # Test `upsert_points` (some out of bound, some updated) + # Test `update_points_from_2d` (some out of bound, some updated) value = 200 oob_value = 5000 assert oob_value > min_bound @@ -829,7 +829,7 @@ def test_instance_group_update_points_from_2d( ) # Not updated assert np.all(instance_group_numpy[~oob_mask_1d_expanded] == value) # Updated - # Test `upsert_points` (between x,y bounds, some out of bound, some updated) + # Test `update_points_from_2d` (between x,y bounds, some out of bound, some updated) value = 300 points = np.full((n_cameras, n_nodes, n_coords), value) # Reset the points to all in bounds From 956e1104a51453f56878994b926f882ef2a86bfa Mon Sep 17 00:00:00 2001 From: roomrys <38435167+roomrys@users.noreply.github.com> Date: Thu, 23 Jan 2025 12:05:33 -0800 Subject: [PATCH 21/38] Template for frame group upsert point --- tests/io/test_cameras.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/io/test_cameras.py b/tests/io/test_cameras.py index f6be4fd31..2b86a3fc7 100644 --- a/tests/io/test_cameras.py +++ b/tests/io/test_cameras.py @@ -1087,6 +1087,22 @@ def test_frame_group( assert labeled_frame in frame_group.session.labels.labeled_frames +def test_frame_group_upsert_points( + multiview_min_session_frame_groups: Labels, +): + # Define Initial 3D point array + ... + + # Call upsert points to update all instance groups + ... + + # Now get 2D points from all instance groups + ... + + # Triangulate 2D points to see if the match initial 3D point array + ... + + def test_cameras_are_not_sorted(): """Test that cameras are not sorted in `RecordingSession`. From 674538d226c24d1e76347c9a9c637c9d4d04fe44 Mon Sep 17 00:00:00 2001 From: roomrys <38435167+roomrys@users.noreply.github.com> Date: Thu, 23 Jan 2025 12:07:29 -0800 Subject: [PATCH 22/38] Pass 3D points in triangulate command --- sleap/gui/commands.py | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/sleap/gui/commands.py b/sleap/gui/commands.py index 03f183a49..cb0080467 100644 --- a/sleap/gui/commands.py +++ b/sleap/gui/commands.py @@ -3772,20 +3772,10 @@ def do_action(cls, context: CommandContext, params: dict): calib=session.camera_cluster, excluded_views=frame_group.excluded_views, ) # F x T x N x 3 - - # Reproject onto all views - pts_reprojected = reproject( - points_3d, - calib=session.camera_cluster, - excluded_views=frame_group.excluded_views, - ) # M=include x F=1 x T x N x 2 - - # Sqeeze back to the original shape - points_reprojected = np.squeeze(pts_reprojected, axis=1) # M=include x TxNx2 - # Update or create/insert ("upsert") instance points + frame_group.upsert_points( - points_3d=points_reprojected, + points=points_3d, instance_groups=instance_groups, exclude_complete=True, ) From 343ff0d524c62524eaa47f24b1418de1cc6db40f Mon Sep 17 00:00:00 2001 From: Jvshen Date: Mon, 27 Jan 2025 14:36:49 -0800 Subject: [PATCH 23/38] Add test for frame_group_upsert_points --- tests/io/test_cameras.py | 34 +++++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/tests/io/test_cameras.py b/tests/io/test_cameras.py index 2b86a3fc7..4bba88539 100644 --- a/tests/io/test_cameras.py +++ b/tests/io/test_cameras.py @@ -1090,18 +1090,42 @@ def test_frame_group( def test_frame_group_upsert_points( multiview_min_session_frame_groups: Labels, ): + from sleap_anipose import reproject, triangulate # Define Initial 3D point array - ... + labels = multiview_min_session_frame_groups + session: RecordingSession = labels.sessions[0] + frame_idx = 0 + frame_group = session.frame_groups[frame_idx] + instance_group = frame_group.instance_groups[0] + + n_cameras = len(frame_group.cams_to_include) + n_nodes = len(frame_group.session.labels.skeleton.nodes) + n_coords = 3 + value = 100 + points = np.full((n_cameras, n_nodes, n_coords), value) - # Call upsert points to update all instance groups - ... + frame_group.upsert_points( + points=points, instance_groups=frame_group.instance_groups, exclude_complete=False + ) # Now get 2D points from all instance groups - ... + instance_group_2d_points = [] + for instance_group in frame_group.instance_groups: + points_2d = instance_group.numpy(invisible_as_nan=True) + instance_group_2d_points.append(points_2d) + + for points_2d in instance_group_2d_points: + assert points_2d.shape[1:] == (n_nodes, 2) # Shape matches nodes × 2D coordinates # Triangulate 2D points to see if the match initial 3D point array - ... + triangulated_points = triangulate( + p2d=np.array(points_2d), + calib=session.camera_cluster, + excluded_views=frame_group.excluded_views, + ) + assert triangulated_points.shape == (1, n_nodes, n_coords) + assert np.allclose(triangulated_points[0], points, atol=1e-2) def test_cameras_are_not_sorted(): """Test that cameras are not sorted in `RecordingSession`. From 675964c0c5b7e2332f368a97300a9f8616cca750 Mon Sep 17 00:00:00 2001 From: Jvshen Date: Tue, 28 Jan 2025 09:47:40 -0800 Subject: [PATCH 24/38] Update doctring for instance_groups input --- sleap/io/cameras.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/sleap/io/cameras.py b/sleap/io/cameras.py index 449c495d8..314ca4dd2 100644 --- a/sleap/io/cameras.py +++ b/sleap/io/cameras.py @@ -2372,14 +2372,14 @@ def upsert_points( Included cams are specified by `FrameGroup.cams_to_include`. The ordering of the `InstanceGroup`s in `instance_groups` should match the - ordering of the second dimension (T) in `points`. + ordering of the first dimension (I) in `points`. Args: - points: Numpy array of shape (M, T, N, 2) where M is the number of views, T - is the number of Tracks, N is the number of Nodes, and 2 is for x, y. - instance_groups: List of `InstanceGroup` objects to update points for. - exclude_complete: If True, then only update points that are not marked as - complete. Default is True. + points: Numpy array of shape (I, N, 3) where I is the number of Instance + groups, N is the number of Nodes, and 3 is for x, y, z. + instance_groups: List of `InstanceGroup` objects to update points for. + exclude_complete: If True, then only update points that are not marked as + complete. Default is True. """ # Ensure we are working with a float array From d9e20375b18485b3a087837b2972c0a73bf790c9 Mon Sep 17 00:00:00 2001 From: Jvshen Date: Tue, 28 Jan 2025 09:48:21 -0800 Subject: [PATCH 25/38] Add checks for correct shape of points --- sleap/io/cameras.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/sleap/io/cameras.py b/sleap/io/cameras.py index 314ca4dd2..0f22a7f83 100644 --- a/sleap/io/cameras.py +++ b/sleap/io/cameras.py @@ -2381,7 +2381,20 @@ def upsert_points( exclude_complete: If True, then only update points that are not marked as complete. Default is True. """ - + points_shape = points.shape + try: + num_instance_groups, num_nodes, num_dims = points_shape + if num_instance_groups != len(instance_groups): + raise ValueError() + if num_dims != 3: + raise ValueError() + except ValueError as e: + raise ValueError( + f"Expected points to have shape (I, N, 3) but got shape {points_shape} " + "where I is the number of Instance groups and N is the number of Nodes." + f"\n{e}" + ) + # Ensure we are working with a float array points = points.astype(float) From 0d959fe17e4a8d3bd5df35de6a5f806f98bd3e20 Mon Sep 17 00:00:00 2001 From: Jvshen Date: Tue, 28 Jan 2025 09:49:10 -0800 Subject: [PATCH 26/38] Fix tests for upsert points --- tests/io/test_cameras.py | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/tests/io/test_cameras.py b/tests/io/test_cameras.py index 4bba88539..3e371577d 100644 --- a/tests/io/test_cameras.py +++ b/tests/io/test_cameras.py @@ -1098,34 +1098,30 @@ def test_frame_group_upsert_points( frame_group = session.frame_groups[frame_idx] instance_group = frame_group.instance_groups[0] - n_cameras = len(frame_group.cams_to_include) + n_instances = len(frame_group.instance_groups) n_nodes = len(frame_group.session.labels.skeleton.nodes) n_coords = 3 value = 100 - points = np.full((n_cameras, n_nodes, n_coords), value) + points = np.full((n_instances, n_nodes, n_coords), value) + points[:, :, :-1] = -80 + points[:, :, -1] = 1100 frame_group.upsert_points( points=points, instance_groups=frame_group.instance_groups, exclude_complete=False ) - # Now get 2D points from all instance groups - instance_group_2d_points = [] - for instance_group in frame_group.instance_groups: - points_2d = instance_group.numpy(invisible_as_nan=True) - instance_group_2d_points.append(points_2d) - - for points_2d in instance_group_2d_points: - assert points_2d.shape[1:] == (n_nodes, 2) # Shape matches nodes × 2D coordinates - # Triangulate 2D points to see if the match initial 3D point array + frame_group_numpy = frame_group.numpy(invisible_as_nan=False) + frame_group_numpy = np.expand_dims(frame_group_numpy, axis=1) triangulated_points = triangulate( - p2d=np.array(points_2d), + p2d=frame_group_numpy, calib=session.camera_cluster, excluded_views=frame_group.excluded_views, ) - assert triangulated_points.shape == (1, n_nodes, n_coords) - assert np.allclose(triangulated_points[0], points, atol=1e-2) + triangulated_points = np.squeeze(triangulated_points, axis=0) + assert triangulated_points.shape == points.shape + assert np.allclose(triangulated_points, points, atol=1e-2) def test_cameras_are_not_sorted(): """Test that cameras are not sorted in `RecordingSession`. From 895688bae00d39e8e9d1e38accb2512213bca970 Mon Sep 17 00:00:00 2001 From: Jvshen Date: Tue, 28 Jan 2025 10:00:41 -0800 Subject: [PATCH 27/38] Set triangulation attribute equal to points --- sleap/io/cameras.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sleap/io/cameras.py b/sleap/io/cameras.py index 0f22a7f83..b0268553b 100644 --- a/sleap/io/cameras.py +++ b/sleap/io/cameras.py @@ -890,6 +890,8 @@ def update_points( f"`CameraCluster` {len(self.camera_cluster)}." ) + self.triangulation = points + # Reproject 3D points into 2D points for each camera view pts_reprojected = reproject( np.expand_dims(points, axis=(0, 1)), # M=include x N x 3 From af9701a007c72838836d87d35e1aab479522eb4a Mon Sep 17 00:00:00 2001 From: Jvshen Date: Tue, 28 Jan 2025 10:01:32 -0800 Subject: [PATCH 28/38] Add test that our triangulation attribute equals points --- tests/io/test_cameras.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/io/test_cameras.py b/tests/io/test_cameras.py index 3e371577d..a6de90120 100644 --- a/tests/io/test_cameras.py +++ b/tests/io/test_cameras.py @@ -721,20 +721,23 @@ def test_instance_group( for inst in instance_group.instances[:2]: lf = inst.frame labels.remove_instance(lf, inst) + points = np.full((n_nodes, 3), 72317) instance_group.update_points( - points=np.full((n_nodes, 3), 72317), + points=points, projection_bounds=projection_bounds, exclude_complete=False, ) + np.testing.assert_equal(instance_group.triangulation, points) for inst in instance_group.instances: if isinstance(inst, PredictedInstance): assert inst.score == instance_group.score prev_score = instance_group.score instance_group.update_points( - points=np.full((n_nodes, 3), 72317), + points=points, projection_bounds=projection_bounds, exclude_complete=False, ) + np.testing.assert_equal(instance_group.triangulation, points) for inst in instance_group.instances: if isinstance(inst, PredictedInstance): assert inst.score == instance_group.score From 92063540bd642bb2743255b07d5e93f50a0e26f7 Mon Sep 17 00:00:00 2001 From: Jvshen Date: Tue, 28 Jan 2025 13:33:18 -0800 Subject: [PATCH 29/38] Squeeze points_3d to be of I x N x 3 dimension --- sleap/gui/commands.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sleap/gui/commands.py b/sleap/gui/commands.py index cb0080467..4cb8b8dca 100644 --- a/sleap/gui/commands.py +++ b/sleap/gui/commands.py @@ -3772,6 +3772,9 @@ def do_action(cls, context: CommandContext, params: dict): calib=session.camera_cluster, excluded_views=frame_group.excluded_views, ) # F x T x N x 3 + + points_3d = np.squeeze(points_3d, axis=0) # I x N x 3 + # Update or create/insert ("upsert") instance points frame_group.upsert_points( From 9623d2977e3a5cc8f38af067f1536565663c21f6 Mon Sep 17 00:00:00 2001 From: Jvshen Date: Tue, 28 Jan 2025 14:11:18 -0800 Subject: [PATCH 30/38] black commands.py --- sleap/gui/commands.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sleap/gui/commands.py b/sleap/gui/commands.py index 4cb8b8dca..2ae68ed9c 100644 --- a/sleap/gui/commands.py +++ b/sleap/gui/commands.py @@ -3772,8 +3772,8 @@ def do_action(cls, context: CommandContext, params: dict): calib=session.camera_cluster, excluded_views=frame_group.excluded_views, ) # F x T x N x 3 - - points_3d = np.squeeze(points_3d, axis=0) # I x N x 3 + + points_3d = np.squeeze(points_3d, axis=0) # I x N x 3 # Update or create/insert ("upsert") instance points From e39fb089e63df2912525fd02491727684fcac33d Mon Sep 17 00:00:00 2001 From: Jvshen Date: Tue, 28 Jan 2025 14:12:36 -0800 Subject: [PATCH 31/38] black cameras.py --- sleap/io/cameras.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/sleap/io/cameras.py b/sleap/io/cameras.py index b0268553b..1122b294d 100644 --- a/sleap/io/cameras.py +++ b/sleap/io/cameras.py @@ -891,7 +891,7 @@ def update_points( ) self.triangulation = points - + # Reproject 3D points into 2D points for each camera view pts_reprojected = reproject( np.expand_dims(points, axis=(0, 1)), # M=include x N x 3 @@ -2377,10 +2377,10 @@ def upsert_points( ordering of the first dimension (I) in `points`. Args: - points: Numpy array of shape (I, N, 3) where I is the number of Instance - groups, N is the number of Nodes, and 3 is for x, y, z. - instance_groups: List of `InstanceGroup` objects to update points for. - exclude_complete: If True, then only update points that are not marked as + points: Numpy array of shape (I, N, 3) where I is the number of Instance + groups, N is the number of Nodes, and 3 is for x, y, z. + instance_groups: List of `InstanceGroup` objects to update points for. + exclude_complete: If True, then only update points that are not marked as complete. Default is True. """ points_shape = points.shape @@ -2396,7 +2396,7 @@ def upsert_points( "where I is the number of Instance groups and N is the number of Nodes." f"\n{e}" ) - + # Ensure we are working with a float array points = points.astype(float) From b861bbe7c32c20ac09985841007b90197c6e36b3 Mon Sep 17 00:00:00 2001 From: Jvshen Date: Tue, 28 Jan 2025 14:13:04 -0800 Subject: [PATCH 32/38] black test_cameras.py --- tests/io/test_cameras.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/io/test_cameras.py b/tests/io/test_cameras.py index a6de90120..bddbe31ee 100644 --- a/tests/io/test_cameras.py +++ b/tests/io/test_cameras.py @@ -1092,8 +1092,9 @@ def test_frame_group( def test_frame_group_upsert_points( multiview_min_session_frame_groups: Labels, -): +): from sleap_anipose import reproject, triangulate + # Define Initial 3D point array labels = multiview_min_session_frame_groups session: RecordingSession = labels.sessions[0] @@ -1110,8 +1111,10 @@ def test_frame_group_upsert_points( points[:, :, -1] = 1100 frame_group.upsert_points( - points=points, instance_groups=frame_group.instance_groups, exclude_complete=False - ) + points=points, + instance_groups=frame_group.instance_groups, + exclude_complete=False, + ) # Triangulate 2D points to see if the match initial 3D point array frame_group_numpy = frame_group.numpy(invisible_as_nan=False) @@ -1126,6 +1129,7 @@ def test_frame_group_upsert_points( assert triangulated_points.shape == points.shape assert np.allclose(triangulated_points, points, atol=1e-2) + def test_cameras_are_not_sorted(): """Test that cameras are not sorted in `RecordingSession`. From 8afe46fab8d399b0a1d2260a8b69277be2e8d489 Mon Sep 17 00:00:00 2001 From: Jvshen Date: Tue, 4 Feb 2025 13:19:43 -0800 Subject: [PATCH 33/38] Change format --- sleap/io/cameras.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/sleap/io/cameras.py b/sleap/io/cameras.py index 1122b294d..1bfa09959 100644 --- a/sleap/io/cameras.py +++ b/sleap/io/cameras.py @@ -885,9 +885,10 @@ def update_points( if len(cams_to_include) + len(excluded_views) != len(self.camera_cluster): raise ValueError( - f"The number of `Camcorder`s to include {len(cams_to_include)} plus the number of `Camcorder`s " - f"to exclude {len(excluded_views)} does not match the number of `Camcorder`s in the " - f"`CameraCluster` {len(self.camera_cluster)}." + f"The number of `Camcorder`s to include {len(cams_to_include)} plus " + f"the number of `Camcorder`s to exclude {len(excluded_views)} does not " + f"match the number of `Camcorder`s in the `CameraCluster` " + f"{len(self.camera_cluster)}." ) self.triangulation = points From c97c8a28c91f864f3b3c56101f698cde2a9ff5f7 Mon Sep 17 00:00:00 2001 From: Jvshen Date: Tue, 4 Feb 2025 13:19:59 -0800 Subject: [PATCH 34/38] Add docstring for update_points_from_2d --- sleap/io/cameras.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/sleap/io/cameras.py b/sleap/io/cameras.py index 1bfa09959..02e5864f9 100644 --- a/sleap/io/cameras.py +++ b/sleap/io/cameras.py @@ -917,7 +917,25 @@ def update_points_from_2d( projection_bounds: np.ndarray, cams_to_include: Optional[List[Camcorder]] = None, exclude_complete: bool = True, - ): + ) -> None: + """Update the points in the `Instance` for the specified `Camcorder`s. + + Args: + points_reprojected: Numpy array of shape (M, N, 2) where M is the number of + views, N is the number of Nodes, and 2 is for x, y. + projection_bounds: Numpy array of shape (M, 2) where M is the number of + views and 2 is for the height and width of the video. + cams_to_include: List of `Camcorder`s to include in the update. The order of + the `Camcorder`s in the list should match the order of the views in the + `points` array. If None, then all `Camcorder`s in the `CameraCluster` + are included. Default is None. + exclude_complete: If True, then do not update points that are marked as + complete. Default is True. + """ + + # If no `Camcorder`s specified, then update `Instance`s for all `CameraCluster` + if cams_to_include is None: + cams_to_include = self.camera_cluster.cameras # Check that correct shape was passed in points_shape = points_reprojected.shape From dbe96740301798b7d795095710ea9fdb9d344ea9 Mon Sep 17 00:00:00 2001 From: Jvshen Date: Tue, 4 Feb 2025 13:20:33 -0800 Subject: [PATCH 35/38] Change location of call --- sleap/io/cameras.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/sleap/io/cameras.py b/sleap/io/cameras.py index 02e5864f9..d0ca1facb 100644 --- a/sleap/io/cameras.py +++ b/sleap/io/cameras.py @@ -957,10 +957,6 @@ def update_points_from_2d( # Ensure we are working with a float array points_reprojected = points_reprojected.astype(np.float64) - # If no `Camcorder`s specified, then update `Instance`s for all `CameraCluster` - if cams_to_include is None: - cams_to_include = self.camera_cluster.cameras - # Get projection bounds (based on video height/width) bounds = projection_bounds # TODO: make sure projection bounds are the shape they need to be in update points bounds_expanded_x = bounds[:, None, 0] From e3ed548c9584cd0d010a3205f894260ecfa2455e Mon Sep 17 00:00:00 2001 From: Jvshen Date: Tue, 4 Feb 2025 13:21:29 -0800 Subject: [PATCH 36/38] Add tests for update_points when it fails --- tests/io/test_cameras.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/io/test_cameras.py b/tests/io/test_cameras.py index bddbe31ee..b129b984c 100644 --- a/tests/io/test_cameras.py +++ b/tests/io/test_cameras.py @@ -716,6 +716,29 @@ def test_instance_group( projection_bounds = np.full((len(instance_group.camera_cluster), 2), np.nan) # Test `update_points` method + + # Test should fail since n_coords != 3 + points = np.full((n_nodes, 2), 72317) + with pytest.raises(ValueError): + instance_group.update_points( + points=points, + projection_bounds=projection_bounds, + exclude_complete=True, + ) + + # Test should fail since len(cams_to_include) + len(excluded_views) != + # len(self.camera_cluster) + points = np.full((n_nodes, 3), 72317) + cams_to_include, excluded_views = [], () + with pytest.raises(ValueError): + instance_group.update_points( + points=points, + projection_bounds=projection_bounds, + exclude_complete=True, + cams_to_include=cams_to_include, + excluded_views=excluded_views, + ) + assert not np.all(instance_group.numpy(invisible_as_nan=False) == 72317) # Remove some Instances to "expose" underlying PredictedInstances for inst in instance_group.instances[:2]: From ff65ab09f58be2dc803b161c7965a98a1bf5e6d6 Mon Sep 17 00:00:00 2001 From: Jvshen Date: Tue, 4 Feb 2025 13:21:55 -0800 Subject: [PATCH 37/38] Add tests for update_points_from_2d when it fails --- tests/io/test_cameras.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/tests/io/test_cameras.py b/tests/io/test_cameras.py index b129b984c..a0351e09c 100644 --- a/tests/io/test_cameras.py +++ b/tests/io/test_cameras.py @@ -799,6 +799,39 @@ def test_instance_group_update_points_from_2d( frame_group = session.frame_groups[frame_idx] instance_group = frame_group.instance_groups[0] + # Call should fail since n_views != len(cams_to_include) + points_reprojected = np.full((0, 1, 1), 0) + projection_bounds = frame_group.session.projection_bounds + cams_to_include = frame_group.cams_to_include + with pytest.raises(ValueError): + instance_group.update_points_from_2d( + points_reprojected=points_reprojected, + projection_bounds=projection_bounds, + exclude_complete=True, + cams_to_include=cams_to_include, + ) + + # Call should fail since n_coords != 2 + points_reprojected = np.full((len(cams_to_include), 1, 1), 0) + with pytest.raises(ValueError): + instance_group.update_points_from_2d( + points_reprojected=points_reprojected, + projection_bounds=projection_bounds, + exclude_complete=True, + cams_to_include=cams_to_include, + ) + + # Verify that points were updated for all cameras in the camera cluster + n_nodes = len(frame_group.session.labels.skeleton.nodes) + points_reprojected = np.random.rand(len(session.camera_cluster.cameras), n_nodes, 2) + projection_bounds = np.array([[1000, 1000]] * len(session.camera_cluster.cameras)) + instance_group.update_points_from_2d( + points_reprojected=points_reprojected, + projection_bounds=projection_bounds, + cams_to_include=None, + exclude_complete=False, + ) + # Test `update_points_from_2d` (all in bounds, all updated) n_cameras = len(frame_group.cams_to_include) n_instance_groups = 1 From eb069ecdf15fa18a5faf6b13e9e8ccb5ab767812 Mon Sep 17 00:00:00 2001 From: Jvshen Date: Tue, 4 Feb 2025 13:22:26 -0800 Subject: [PATCH 38/38] Add tests for upsert_points when it fails --- tests/io/test_cameras.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/io/test_cameras.py b/tests/io/test_cameras.py index a0351e09c..ef482eb45 100644 --- a/tests/io/test_cameras.py +++ b/tests/io/test_cameras.py @@ -1172,6 +1172,25 @@ def test_frame_group_upsert_points( exclude_complete=False, ) + # Call should fail since num_instance_groups != len(instance_groups) + instance_groups = frame_group.instance_groups + points_incorrect_shape = np.full((len(instance_groups) + 1, 2, 3), 0) + with pytest.raises(ValueError): + frame_group.upsert_points( + points=points_incorrect_shape, + instance_groups=instance_groups, + exclude_complete=False, + ) + + # Call should fail since num_dims != 3 + points_incorrect_shape = np.full((len(instance_groups), 2, 2), 0) + with pytest.raises(ValueError): + frame_group.upsert_points( + points=points_incorrect_shape, + instance_groups=instance_groups, + exclude_complete=False, + ) + # Triangulate 2D points to see if the match initial 3D point array frame_group_numpy = frame_group.numpy(invisible_as_nan=False) frame_group_numpy = np.expand_dims(frame_group_numpy, axis=1)