update saving to svo file using video converter

configurations/sam2_zed_small.yaml

@@ -7,18 +7,18 @@ grounding_dino:
   nms_threshold: 0.6
   # config_path: "/home/nakama/Documents/TychoMSC/models/sam2_track_test/segment-anything-2-real-time/GroundingDINO/groundingdino/config/GroundingDINO_SwinT_OGC.py"
   config_path: "./GroundingDINO/groundingdino/config/GroundingDINO_SwinT_OGC.py"
-  # checkpoint_path: "/home/nakama/Documents/TychoMSC/models/sam2_track_test/segment-anything-2-real-time/GroundingDINO/weights/groundingdino_swint_ogc.pth"
   checkpoint_path: "./GroundingDINO/weights/groundingdino_swint_ogc.pth"
 
 sam2:
   checkpoint: "./checkpoints/sam2_hiera_small.pt"
   model_cfg: "sam2_configs/sam2_hiera_s.yaml"
 
 camera:
-  connection_type: "id"
+  connection_type: "svo"
 
 depth:
   refine_depth: true
+  max_occlusion_percentage: 0.6
 
 results:
   output_dir: "output"

scripts/sam2_track_zed.py

@@ -33,11 +33,11 @@
 from utils.sam2prty import Sam2PromptType
 
 
-depth_refinement_enabled = False
 
 
 def run(cfg, sam2_prompt: Sam2PromptType) -> None:
-    global depth_refinement_enabled
+    depth_refinement_enabled = cfg.depth.refine_depth
+
     caption_queue = queue.Queue()
 
     Log.info("Initializing the pipeline...", tag="pipeline_init")
@@ -246,7 +246,7 @@ def run(cfg, sam2_prompt: Sam2PromptType) -> None:
 
 
                 left_image_rgb = cv2.addWeighted(left_image_rgb, 1, all_mask, 0.5, 0)
-                if framecount < 500:
+                if framecount < 5000:
                     # left_image_rgb = np.array(left_image_rgb)
                     # norm_depth_map = np.array(norm_depth_map)
                     # matr = np.ones((1440,1280,4), dtype='uint8')
@@ -270,7 +270,7 @@ def run(cfg, sam2_prompt: Sam2PromptType) -> None:
                 # *Refine the depth mask using masks
                 if depth_refinement_enabled:
                     # !NOTE: plane fiting best one so far
-                    refined_depth_map = depth_refinement_RANSAC_plane_fitting(depth_map, obj_masks, max_occ_percentage=0.7)
+                    refined_depth_map = depth_refinement_RANSAC_plane_fitting(depth_map, obj_masks, max_occ_percentage=cfg.depth.max_occlusion_percentage)
                     refined_depth_map = cv2.normalize(refined_depth_map, None, 0, 255, cv2.NORM_MINMAX).astype(np.uint8)
                     # previous_depth = norm_depth_map
                     cv2.imwrite(os.path.join(output_dir, 'refined_depth.png'), refined_depth_map)
@@ -300,7 +300,7 @@ def run(cfg, sam2_prompt: Sam2PromptType) -> None:
 
     with initialize(config_path="../configurations"):
         cfg = compose(config_name="sam2_zed_small")
-        sam2_prompt = Sam2PromptType('g_dino_bbox',user_caption='apple')
+        sam2_prompt = Sam2PromptType('g_dino_bbox',user_caption='keyboard')
 
 
         # point_coords = [(390, 200)]

scripts/sam2_track_zed_video.py

@@ -0,0 +1,121 @@
+from utils.logger import Log
+import os,sys
+import torch
+import numpy as np
+import random
+import cv2
+import time
+from hydra import initialize, compose
+from hydra.core.global_hydra import GlobalHydra
+import imageio
+
+import threading
+import queue
+from utils.sam2prty import Sam2PromptType
+sys.path.insert(0, os.getcwd())
+
+
+# Initialize GPU settings
+if torch.cuda.get_device_properties(0).major >= 8:
+    torch.backends.cuda.matmul.allow_tf32 = True
+    torch.backends.cudnn.allow_tf32 = True
+torch.backends.cudnn.benchmark = True
+torch.autocast(device_type="cuda", dtype=torch.bfloat16).__enter__()
+
+
+from wrappers.pyzed_wrapper import pyzed_wrapper as pw
+from utils.utils import *
+from scripts.utils.depth_utils import (
+    depth_refinement_RANSAC_plane_fitting,
+    )
+
+
+def run(cfg, sam2_prompt: Sam2PromptType, record: bool = False, svo_file: str = None) -> None:
+    depth_refinement_enabled = cfg.depth.refine_depth
+    caption_queue = queue.Queue()
+
+    Log.info("Initializing the pipeline...", tag="pipeline_init")
+
+    # Build needed models
+    Log.info("Building models...", tag="building_models")
+    # try:
+    #     sam2_predictor, grounding_dino_model = build_models(cfg)
+    #     Log.info("Models successfully built and loaded.", tag="model_building")
+    # except Exception as e:
+    #     Log.error(f"Failed to build models: {e}", tag="model_build_error")
+    #     return
+
+    # Initialize ZED camera: Live Mode or SVO Playback Mode
+    Log.info("Initializing ZED camera...", tag="zed_camera_init")
+    wrapper = pw.Wrapper("svo" if svo_file else cfg.camera.connection_type)
+
+    # If using playback, set SVO file path
+    if svo_file:
+        wrapper.input_parameters.svo_input_filename = svo_file
+
+    try:
+        wrapper.open_input_source()
+        Log.info("ZED camera initialized.", tag="camera_init")
+    except Exception as e:
+        Log.error(f"Failed to initialize ZED camera: {e}", tag="camera_init_error")
+        return
+
+    # Start recording if enabled
+    if record:
+        wrapper.start_recording("./output/output.svo")
+        Log.info("Recording started.", tag="recording")
+
+    wrapper.start_stream()
+    Log.info('Camera stream started.', tag = "camera_stream")
+
+    framecount = 0
+
+    try:
+        while True:
+            ts = time.time()
+
+            if wrapper.retrieve(is_image=True, is_measure=True):
+                # Extract images
+                left_image = wrapper.output_image
+                depth_map = wrapper.output_measure
+
+                norm_depth_map = cv2.normalize(depth_map, None, 0, 255, cv2.NORM_MINMAX).astype(np.uint8)
+                left_image_rgb = cv2.cvtColor(left_image, cv2.COLOR_RGBA2RGB)
+
+                # # Display and save
+                # cv2.imshow("ZED Left", left_image_rgb)
+                # cv2.imshow("Depth Map", norm_depth_map)
+
+                # cv2.imwrite(f"output/frame_{framecount}.png", left_image_rgb)
+                # cv2.imwrite(f"output/depth_{framecount}.png", norm_depth_map)
+
+                if framecount > 100:
+                    break
+
+                framecount += 1
+
+    except Exception as e:
+        Log.error(f"An error occurred: {e}", tag="runtime_error")
+
+    finally:
+        # Stop recording if enabled
+        if record:
+            wrapper.stop_recording()
+            Log.info("Recording stopped.", tag="recording_stop")
+
+        Log.info("Shutting down and closing stream...", tag='Close_stream')
+        wrapper.stop_stream()
+        wrapper.close_input_source()
+        cv2.destroyAllWindows()
+
+if __name__ == "__main__":
+    # Ensure GlobalHydra is properly cleared before initializing
+    if GlobalHydra.instance().is_initialized():
+        GlobalHydra.instance().clear()
+
+    with initialize(config_path="../configurations"):
+        cfg = compose(config_name="sam2_zed_small")
+        sam2_prompt = Sam2PromptType('g_dino_bbox', user_caption='keyboard')
+
+    # Run after Hydra is fully initialized
+    run(cfg, sam2_prompt, record=True)

scripts/wrappers/pyzed_wrapper/wrapper_settings.py

@@ -15,7 +15,7 @@ def __init__(self, type="id"):
         elif self.type == self.SERIAL:
             self.serial_number = 0
         elif self.type == self.SVO:
-            self.svo_input_filename = "./output.svo"
+            self.svo_input_filename = "./output/output.svo2"
 
 
         elif self.type == self.STREAM: