added tmp scripts

README.md

@@ -43,6 +43,10 @@ Given a single RGB-D image of transparent objects, ClearGrasp first uses the col
 If you have any questions or find any bugs, please file a github issue or contact me:  
 Shreeyak Sajjan: shreeyak[dot]sajjan[at]gmail[dot]com
 
+### Updates:
+
+- Added temporary scripts that were used in handling data and creating visualizations. Reference only, might not run.
+
 ## Installation
 
 This code is tested with Ubuntu 16.04, Python3.6 and [Pytorch](https://pytorch.org/get-started/locally/) 1.3, and CUDA 9.0.  
@@ -247,6 +251,12 @@ Run the `dataset_capture_gui/capture_image.py` script to launch a window that st
 
 ## FAQ
 
+### Data Preprocessing (outlines creation)
+
+The script used for preprocessing data can be found at `z-ignore-scripts-helper/data_processing_script.py`. The script
+will not run due to missing files (these were intermediate files generated at render time, such as unrectified depth). 
+However, the script can be used as reference for creation of the outlines.
+
 ### Details on depth2depth
 
 The `depth2depth` executable expects the following parameters:

requirements.txt

@@ -3,7 +3,7 @@ h5py>=2.10.0
 imageio>=2.6.1
 git+https://github.com/aleju/imgaug
 numpy>=1.17.3
-opencv-python==4.1.1.26
+opencv-python==4.1.2.30
 git+https://github.com/jamesbowman/openexrpython.git
 oyaml>=0.9
 pathlib>=1.0.1

z-ignore-scripts-helper/.gitignore

@@ -0,0 +1,2 @@
+*.png
+*.ply

z-ignore-scripts-helper/SensorData.py

@@ -0,0 +1,120 @@
+import os, struct
+import numpy as np
+import zlib
+import imageio
+import cv2
+
+COMPRESSION_TYPE_COLOR = {-1:'unknown', 0:'raw', 1:'png', 2:'jpeg'}
+COMPRESSION_TYPE_DEPTH = {-1:'unknown', 0:'raw_ushort', 1:'zlib_ushort', 2:'occi_ushort'}
+
+class RGBDFrame():
+
+  def load(self, file_handle):
+    self.camera_to_world = np.asarray(struct.unpack('f'*16, file_handle.read(16*4)), dtype=np.float32).reshape(4, 4)
+    self.timestamp_color = struct.unpack('Q', file_handle.read(8))[0]
+    self.timestamp_depth = struct.unpack('Q', file_handle.read(8))[0]
+    self.color_size_bytes = struct.unpack('Q', file_handle.read(8))[0]
+    self.depth_size_bytes = struct.unpack('Q', file_handle.read(8))[0]
+    self.color_data = b''.join(struct.unpack('c'*self.color_size_bytes, file_handle.read(self.color_size_bytes)))
+    self.depth_data = b''.join(struct.unpack('c'*self.depth_size_bytes, file_handle.read(self.depth_size_bytes)))
+
+
+  def decompress_depth(self, compression_type):
+    if compression_type == 'zlib_ushort':
+       return self.decompress_depth_zlib()
+    else:
+       raise
+
+
+  def decompress_depth_zlib(self):
+    return zlib.decompress(self.depth_data)
+
+
+  def decompress_color(self, compression_type):
+    if compression_type == 'jpeg':
+       return self.decompress_color_jpeg()
+    else:
+       raise
+
+
+  def decompress_color_jpeg(self):
+    return imageio.imread(self.color_data)
+
+
+class SensorData:
+
+  def __init__(self, filename):
+    self.version = 4
+    self.load(filename)
+
+
+  def load(self, filename):
+    with open(filename, 'rb') as f:
+      version = struct.unpack('I', f.read(4))[0]
+      assert self.version == version
+      strlen = struct.unpack('Q', f.read(8))[0]
+      self.sensor_name = b''.join(struct.unpack('c'*strlen, f.read(strlen)))
+      self.intrinsic_color = np.asarray(struct.unpack('f'*16, f.read(16*4)), dtype=np.float32).reshape(4, 4)
+      self.extrinsic_color = np.asarray(struct.unpack('f'*16, f.read(16*4)), dtype=np.float32).reshape(4, 4)
+      self.intrinsic_depth = np.asarray(struct.unpack('f'*16, f.read(16*4)), dtype=np.float32).reshape(4, 4)
+      self.extrinsic_depth = np.asarray(struct.unpack('f'*16, f.read(16*4)), dtype=np.float32).reshape(4, 4)
+      self.color_compression_type = COMPRESSION_TYPE_COLOR[struct.unpack('i', f.read(4))[0]]
+      self.depth_compression_type = COMPRESSION_TYPE_DEPTH[struct.unpack('i', f.read(4))[0]]
+      self.color_width = struct.unpack('I', f.read(4))[0]
+      self.color_height =  struct.unpack('I', f.read(4))[0]
+      self.depth_width = struct.unpack('I', f.read(4))[0]
+      self.depth_height =  struct.unpack('I', f.read(4))[0]
+      self.depth_shift =  struct.unpack('f', f.read(4))[0]
+      num_frames =  struct.unpack('Q', f.read(8))[0]
+      self.frames = []
+      for i in range(num_frames):
+        frame = RGBDFrame()
+        frame.load(f)
+        self.frames.append(frame)
+
+
+  def export_depth_images(self, output_path, image_size=None, frame_skip=1):
+    if not os.path.exists(output_path):
+      os.makedirs(output_path)
+    print('exporting', len(self.frames)//frame_skip, ' depth frames to', output_path)
+    for f in range(0, len(self.frames), frame_skip):
+      depth_data = self.frames[f].decompress_depth(self.depth_compression_type)
+      depth = np.fromstring(depth_data, dtype=np.uint16).reshape(self.depth_height, self.depth_width)
+      if image_size is not None:
+        depth = cv2.resize(depth, (image_size[1], image_size[0]), interpolation=cv2.INTER_NEAREST)
+      imageio.imwrite(os.path.join(output_path, str(f) + '.png'), depth)
+
+
+  def export_color_images(self, output_path, image_size=None, frame_skip=1):
+    if not os.path.exists(output_path):
+      os.makedirs(output_path)
+    print('exporting', len(self.frames)//frame_skip, 'color frames to', output_path)
+    for f in range(0, len(self.frames), frame_skip):
+      color = self.frames[f].decompress_color(self.color_compression_type)
+      if image_size is not None:
+        color = cv2.resize(color, (image_size[1], image_size[0]), interpolation=cv2.INTER_NEAREST)
+      imageio.imwrite(os.path.join(output_path, str(f) + '.jpg'), color)
+
+
+  def save_mat_to_file(self, matrix, filename):
+    with open(filename, 'w') as f:
+      for line in matrix:
+        np.savetxt(f, line[np.newaxis], fmt='%f')
+
+
+  def export_poses(self, output_path, frame_skip=1):
+    if not os.path.exists(output_path):
+      os.makedirs(output_path)
+    print('exporting', len(self.frames)//frame_skip, 'camera poses to', output_path)
+    for f in range(0, len(self.frames), frame_skip):
+      self.save_mat_to_file(self.frames[f].camera_to_world, os.path.join(output_path, str(f) + '.txt'))
+
+
+  def export_intrinsics(self, output_path):
+    if not os.path.exists(output_path):
+      os.makedirs(output_path)
+    print('exporting camera intrinsics to', output_path)
+    self.save_mat_to_file(self.intrinsic_color, os.path.join(output_path, 'intrinsic_color.txt'))
+    self.save_mat_to_file(self.extrinsic_color, os.path.join(output_path, 'extrinsic_color.txt'))
+    self.save_mat_to_file(self.intrinsic_depth, os.path.join(output_path, 'intrinsic_depth.txt'))
+    self.save_mat_to_file(self.extrinsic_depth, os.path.join(output_path, 'extrinsic_depth.txt'))

z-ignore-scripts-helper/calculate_false_positives.py

@@ -0,0 +1,47 @@
+# Use the numpy library.
+import numpy as np
+
+
+def compute_metrics(pred, label):
+    """Compute metrics like True/False Positive, True/False Negative.`
+        MUST HAVE ONLY 2 CLASSES: BACKGROUND, OBJECT.
+    Args:
+        pred (numpy.ndarray): Prediction, one-hot encoded. Shape: [2, H, W], dtype: uint8
+        label (numpy.ndarray): Ground Truth, one-hot encoded. Shape: [H, W], dtype: uint8
+    Returns:
+        float: IOU, TP, TN, FP, FN
+    """
+    if len(pred.shape) > 3:
+        raise ValueError("pred should have shape [2, H, W], got: {}".format(pred.shape))
+    if len(label.shape) > 2:
+        raise ValueError("label should have shape [H, W], got: {}".format(label.shape))
+
+    total_pixels = pred.shape[0] * pred.shape[1]
+
+    tp = np.sum(np.logical_and(pred == 1, label > 0))
+    tn = np.sum(np.logical_and(pred == 0, label == 0))
+    fp = np.sum(np.logical_and(pred == 1, label == 0))
+    fn = np.sum(np.logical_and(pred == 0, label > 0))
+
+    if (tp + tn + fp + fn) != total_pixels:
+        raise ValueError('The number of total pixels ({}) and sum of tp,fp,tn,fn ({}) is not equal'.format(
+            total_pixels, (tp + tn + fp + fn)))
+    iou = tp / (tp + fp + fn)
+
+    _tp = tp / np.sum(label == 1)
+
+    tp_rate = (tp / (tp + fn)) * 100
+    fp_rate = (fp / (fp + tn)) * 100
+    tn_rate = (tn / (tn + fp)) * 100
+    fn_rate = (fn / (fn + tp)) * 100
+
+    return iou, tp_rate, tn_rate, fp_rate, fn_rate
+
+height, width = 20, 20
+x = np.zeros((height, width))
+y = np.ones((height, width))
+
+x[:10, :10] = 1
+
+iou, tp_rate, tn_rate, fp_rate, fn_rate = compute_metrics(x, y)
+print(iou, tp_rate, tn_rate, fp_rate, fn_rate)