Merge pull request DLR-RM#462 from apenzko/master

Stereo matching with projector example

blenderproc/api/utility/__init__.py

@@ -1,2 +1,3 @@
 from blenderproc.python.utility.Utility import resolve_path, num_frames, resolve_resource, set_keyframe_render_interval, reset_keyframes
-from blenderproc.python.utility.LabelIdMapping import LabelIdMapping
+from blenderproc.python.utility.LabelIdMapping import LabelIdMapping
+from blenderproc.python.utility.PatternUtility import generate_random_pattern_img

blenderproc/python/modules/lighting/LightInterface.py

@@ -63,4 +63,8 @@ def _add_light_source(self, config):
         light.set_rotation_euler(config.get_list("rotation", [0, 0, 0]))
         light.set_energy(config.get_float("energy", 10.))
         light.set_color(config.get_list("color", [1, 1, 1])[:3])
-        light.set_distance(config.get_float("distance", 0))
+        light.set_distance(config.get_float("distance", 0))
+        if config.get_bool("use_projector", False):
+            light.setup_as_projector(config.get_string("path", "examples/advanced/stereo_matching_with_projector"
+                                                               "/patterns/random_pattern_00256.png"))
+

blenderproc/python/types/LightUtility.py

@@ -5,6 +5,7 @@
 from blenderproc.python.types.EntityUtility import Entity
 from blenderproc.python.utility.Utility import Utility, KeyFrame
 from mathutils import Color
+import numpy as np
 
 
 class Light(Entity):
@@ -62,6 +63,109 @@ def set_type(self, type: str, frame: int = None):
         self.blender_obj.data.type = type
         Utility.insert_keyframe(self.blender_obj.data, "type", frame)
 
+    def setup_as_projector(self, pattern: np.ndarray, frame: int = None):
+        """ Sets a spot light source as projector of a pattern image. Sets location and angle of projector to current
+        camera. Adjusts scale of pattern image to fit field-of-view of camera:
+        $(0.5 + \frac{X}{Z \cdot F}, 0.5 + \frac{X}{Z \cdot F \cdot r}, 0)$
+        where $F$ is focal length and $r$ aspect ratio.
+        WARNING: This should be done after the camera parameters are set!
+
+        :param pattern: pattern image to be projected onto scene as np.ndarray.
+        :param frame: The frame number which the value should be set to. If None is given, the current frame number is used.
+        """
+        cam_ob = bpy.context.scene.camera
+        fov = cam_ob.data.angle     # field of view of current camera in radians
+
+        focal_length = 2 * np.tan(fov / 2)
+        # Image aspect ratio = height / width
+        aspect_ratio = bpy.context.scene.render.resolution_y / bpy.context.scene.render.resolution_x
+
+        # Set location of light source to camera -- COPY TRANSFORMS
+        self.blender_obj.constraints.new('COPY_TRANSFORMS')
+        self.blender_obj.constraints['Copy Transforms'].target = cam_ob
+
+        # Setup nodes for projecting image
+        self.blender_obj.data.use_nodes = True
+        self.blender_obj.data.shadow_soft_size = 0
+        self.blender_obj.data.spot_size = 3.14159  # 180deg in rad
+        self.blender_obj.data.cycles.cast_shadow = False
+
+        nodes = self.blender_obj.data.node_tree.nodes
+        links = self.blender_obj.data.node_tree.links
+
+        node_ox = nodes.get('Emission')
+
+        image_data = bpy.data.images.new('pattern', width=pattern.shape[1], height=pattern.shape[0], alpha=True)
+        image_data.pixels = pattern.ravel()
+
+        # Set Up Nodes
+        node_pattern = nodes.new(type="ShaderNodeTexImage")  # Texture Image
+        node_pattern.label = 'Texture Image'
+        node_pattern.image = bpy.data.images['pattern']
+        node_pattern.extension = 'CLIP'
+
+        node_coord = nodes.new(type="ShaderNodeTexCoord")  # Texture Coordinate
+        node_coord.label = 'Texture Coordinate'
+
+        f_value = nodes.new(type="ShaderNodeValue")
+        f_value.label = 'Focal Length'
+        f_value.outputs[0].default_value = focal_length
+
+        fr_value = nodes.new(type="ShaderNodeValue")
+        fr_value.label = 'Focal Length * Ratio'
+        fr_value.outputs[0].default_value = focal_length * aspect_ratio
+
+        divide1 = nodes.new(type="ShaderNodeMath")
+        divide1.label = 'X / ZF'
+        divide1.operation = 'DIVIDE'
+
+        divide2 = nodes.new(type="ShaderNodeMath")
+        divide2.label = 'Y / ZFr'
+        divide2.operation = 'DIVIDE'
+
+        multiply1 = nodes.new(type="ShaderNodeMath")
+        multiply1.label = 'Z * F'
+        multiply1.operation = 'MULTIPLY'
+
+        multiply2 = nodes.new(type="ShaderNodeMath")
+        multiply2.label = 'Z * Fr'
+        multiply2.operation = 'MULTIPLY'
+
+        center_image = nodes.new(type="ShaderNodeVectorMath")
+        center_image.operation = 'ADD'
+        center_image.label = 'Offset'
+        center_image.inputs[1].default_value[0] = 0.5
+        center_image.inputs[1].default_value[1] = 0.5
+
+        xyz_components = nodes.new(type="ShaderNodeSeparateXYZ")
+
+        combine_xyz = nodes.new(type="ShaderNodeCombineXYZ")
+
+        # Set Up Links
+        links.new(node_pattern.outputs["Color"], node_ox.inputs["Color"])  # Link Image Texture to Emission
+        links.new(node_coord.outputs["Normal"], xyz_components.inputs["Vector"])
+        # ZF
+        links.new(f_value.outputs[0], multiply1.inputs[1])
+        links.new(xyz_components.outputs["Z"], multiply1.inputs[0])
+        # ZFr
+        links.new(fr_value.outputs[0], multiply2.inputs[1])
+        links.new(xyz_components.outputs["Z"], multiply2.inputs[0])
+        # X / ZF
+        links.new(xyz_components.outputs["X"], divide1.inputs[0])
+        links.new(multiply1.outputs[0], divide1.inputs[1])
+        # Y / ZFr
+        links.new(xyz_components.outputs["Y"], divide2.inputs[0])
+        links.new(multiply2.outputs[0], divide2.inputs[1])
+        # Combine (X/ZF, Y/ZFr, 0)
+        links.new(divide1.outputs[0], combine_xyz.inputs["X"])
+        links.new(divide2.outputs[0], combine_xyz.inputs["Y"])
+        # Center image by offset
+        links.new(combine_xyz.outputs["Vector"], center_image.inputs[0])
+        # Link Mapping to Image Texture
+        links.new(center_image.outputs["Vector"], node_pattern.inputs["Vector"])
+
+        Utility.insert_keyframe(self.blender_obj.data, "use_projector", frame)
+
 
     def get_energy(self, frame: int = None) -> float:
         """ Returns the energy of the light.

blenderproc/python/utility/PatternUtility.py

@@ -0,0 +1,24 @@
+import numpy as np
+import random
+import cv2
+
+
+def generate_random_pattern_img(width: int, height: int, n_points: int) -> np.ndarray:
+    """Generate transparent image with random pattern.
+
+    :param width: width of image to be generated.
+    :param height: height of image to be generated.
+    :param n_points: number of white points uniformly placed on image.
+    """
+    pattern_img = np.zeros((height, width, 4), dtype=np.uint8)
+
+    m_width = int(width // np.sqrt(n_points))
+    m_height = int(height // np.sqrt(n_points))
+
+    for i in range(width // m_width):
+        for j in range(height // m_height):
+            x_idx = random.randint(i * m_width, (i + 1) * m_width - 1)
+            y_idx = random.randint(j * m_height, (j + 1) * m_height - 1)
+            pattern_img = cv2.circle(pattern_img, (x_idx, y_idx), 1, (255, 255, 255, 255), -1)
+
+    return pattern_img

examples/advanced/stereo_matching_with_projector/README.md

@@ -0,0 +1,79 @@
+# Stereo Matching with Random Pattern Projector 
+![](../../../images/stereo_with_projector.png)
+
+On the left side we can see the rendered RGB image (right) without a pattern and the corresponding stereo depth estimation on the bottom. 
+On the right side we can see the rendered RGB image (right) with a projected random pattern adding 25600 points to the image and the corresponding stereo depth estimation on the bottom. 
+Adding a random pattern to the image increases available features for the stereo matching algorithm, making it easier to discern small details, such as the chair's arm rest. 
+Furthermore, the added random pattern is projected through a SPOT light source in blender, which loses intensity the further away from the source it gets. 
+
+## Usage
+
+Execute in the BlenderProc main directory:
+
+```
+blenderproc run examples/advanced/stereo_matching_with_projector/main.py <path to cam_pose file> <path to house.json> examples/advanced/stereo_matching/output <number of points>
+```
+
+* `examples/advanced/stereo_matching_with_projector/main.py`: path to the main python file to run.
+* `<path to cam_pose file>`: Should point to a file which describes one camera pose per line (here the output of `scn2cam` from the `SUNCGToolbox` can be used).
+* `<path to house.json>`: Path to the house.json file of the SUNCG scene you want to render. Which should be either located inside the SUNCG directory, or the SUNCG directory path should be added to the config file.
+* `<number of points>`: Number of points for random pattern. Default = 2560.
+* `examples/advanced/stereo_matching_with_projector/output`: path to the output directory.
+
+## Visualizaton
+Visualize the generated data:
+```
+blenderproc vis hdf5 examples/advanced/stereo_matching_with_projector/output/0.hdf5
+```
+
+## Implementation
+
+```python
+# Genrate pattern image
+pattern_img = bproc.utility.generate_random_pattern_img(WIDTH, HEIGHT, args.points)
+
+# Define a new light source and set it as projector
+light = bproc.types.Light()
+light.set_type('SPOT')
+light.set_energy(3000)
+fov = bproc.camera.get_fov()
+ratio = HEIGHT / WIDTH
+light.setup_as_projector(fov[0], ratio, pattern_img)
+```
+Here we setup the projector: 
+* Generate a pattern image to be projected onto the scene. 
+* Set new spot light with custom energy. 
+* Using the spot light as projector for specified pattern image `pattern_img`: 
+  * Set projector location to camera via `COPY TRANSFORMS`
+  * Scale image based on field of view `fov` of camera and rendered image size `WIDTH, HEIGHT`
+  * Link image as texture
+  > for further details see implementation at [blenderproc/python/types/LightUtility.py](blenderproc/python/types/LightUtility.py)
+
+```python
+# Enable stereo mode and set baseline
+bproc.camera.set_stereo_parameters(interocular_distance=0.05, convergence_mode="PARALLEL")
+```
+
+Here we enable stereo rendering and specify the camera parameters, some notable points are:
+* Setting the `interocular_distance` which is the stereo baseline.
+* Specifying `convergence_mode` to be `"PARALLEL"` (i.e. both cameras lie on the same line and are just shifted by `interocular_distance`, and are trivially coplanar).
+    * Other options are `OFF-AXIS` where the cameras rotate inwards (converge) up to some plane.  
+  * `convergence_distance` is the distance from the cameras to the aforementioned plane they converge to in case of `OFF-AXIS` convergence mode. In this case, this parameter is ignored by Blender, but it is added here for clarification.
+
+```python
+# Apply stereo matching to each pair of images
+data["stereo-depth"], data["disparity"] = bproc.postprocessing.stereo_global_matching(data["colors"], disparity_filter=False)
+```
+
+Here we apply the stereo matching.
+* It is based on OpenCV's [implementation](https://docs.opencv.org/2.4/modules/calib3d/doc/camera_calibration_and_3d_reconstruction.html?highlight=sgbm#stereosgbm-stereosgbm) of [stereo semi global matching](https://elib.dlr.de/73119/1/180Hirschmueller.pdf).
+* Its pipeline runs as follows:
+    * Compute the disparity map between the two images. After specifying the required parameters.
+    * Optional use of a disparity filter (namely `wls_filter`). Enabled by setting `disparity_filter` (Enabling it could possibly lead to less accurate depth values. One should experiment with this parameter).
+    * Triangulate the depth values using the focal length and disparity.
+    * Clip the depth map from 0 to `depth_max`, where this value is retrieved from `renderer.Renderer`.
+    * Apply an optional [depth completion routine](https://github.com/kujason/ip_basic/blob/master/ip_basic/depth_map_utils.py), based on simple image processing techniques. This is enabled by setting `depth_completion`.
+* There are some stereo semi global matching parameters that can be tuned (see fct docs), such as:
+    * `window_size`
+    * `num_disparities`
+    * `min_disparity`

examples/advanced/stereo_matching_with_projector/config.yaml

@@ -0,0 +1,86 @@
+# Args: <cam_file> <obj_file> <pattern_path> <output_dir>
+{
+  "version": 3,
+  "setup": {
+    "blender_install_path": "/home_local/<env:USER>/blender/",
+    "pip": [
+      "h5py",
+      "python-dateutil==2.1",
+      "numpy",
+      "Pillow",
+      "opencv-contrib-python",
+      "scipy"
+    ]
+  },
+  "modules": [
+    {
+      "module": "main.Initializer",
+      "config": {
+        "global": {
+          "output_dir": "<args:3>"
+        }
+      }
+    },
+    {
+      "module": "loader.SuncgLoader",
+      "config": {
+        "path": "<args:1>"
+      }
+    },
+    {
+      "module": "camera.CameraLoader",
+      "config": {
+        "path": "<args:0>",
+        "file_format": "location rotation/value _ _ _ _ _ _",
+        "world_frame_change": ["X", "-Z", "Y"],
+        "default_cam_param": {
+          "rotation": {
+            "format": "forward_vec"
+          }
+        },
+        "intrinsics": {
+          "interocular_distance": 0.05,
+          "stereo_convergence_mode": "PARALLEL",
+          "convergence_distance": 0.00001,
+          "cam_K": [650.018, 0, 637.962, 0, 650.018, 355.984, 0, 0 ,1],
+          "resolution_x": 1280,
+          "resolution_y": 720
+        },
+      }
+    },
+    { # Projector Light (needs to be after camera loader)
+      "module": "lighting.LightLoader",
+      "config": {
+        "lights": [
+          {
+            "type": "SPOT",
+            "energy": 3000,
+            "use_projector": True,
+            "path": "<args:2>"
+          }
+        ]
+      }
+    },
+    {
+      "module": "lighting.SuncgLighting",
+      "config": {}
+    },
+    {
+      "module": "renderer.RgbRenderer",
+      "config": {
+        "render_distance": true,
+        "stereo": true,
+        "use_alpha": true,
+      }
+    },
+    {
+      "module": "writer.StereoGlobalMatchingWriter",
+      "config": {
+        "disparity_filter": false
+      }
+    },
+    {
+      "module": "writer.Hdf5Writer",
+    }
+  ]
+}

examples/advanced/stereo_matching_with_projector/main.py

@@ -0,0 +1,64 @@
+import blenderproc as bproc
+import argparse
+import os
+import numpy as np
+
+parser = argparse.ArgumentParser()
+parser.add_argument('camera', help="Path to the camera file which describes one camera pose per line, here the output of scn2cam from the SUNCGToolbox can be used")
+parser.add_argument('house', help="Path to the house.json file of the SUNCG scene to load")
+parser.add_argument('output_dir', nargs='?', default="examples/datasets/suncg_basic/output", help="Path to where the final files, will be saved")
+parser.add_argument('points', type=int, default=2560, help="Number of points for random pattern. Not always exact due to rounding errors.")
+args = parser.parse_args()
+
+bproc.init()
+
+# load the objects into the scene
+label_mapping = bproc.utility.LabelIdMapping.from_csv(bproc.utility.resolve_resource(os.path.join('id_mappings', 'nyu_idset.csv')))
+objs = bproc.loader.load_suncg(args.house, label_mapping=label_mapping)
+
+# define the camera intrinsics
+K = np.array([
+    [650.018, 0, 637.962],
+    [0, 650.018, 355.984],
+    [0, 0, 1]
+])
+WIDTH, HEIGHT = 1280, 720
+bproc.camera.set_intrinsics_from_K_matrix(K, WIDTH, HEIGHT)
+
+# Enable stereo mode and set baseline
+bproc.camera.set_stereo_parameters(interocular_distance=0.05, convergence_mode="PARALLEL", convergence_distance=0.00001)
+
+# Genrate pattern image
+pattern_img = bproc.utility.generate_random_pattern_img(WIDTH, HEIGHT, args.points)
+
+# Define a new light source and set it as projector
+light = bproc.types.Light()
+light.set_type('SPOT')
+light.set_energy(3000)
+light.setup_as_projector(pattern_img)
+
+# read the camera positions file and convert into homogeneous camera-world transformation
+with open(args.camera, "r") as f:
+    for line in f.readlines():
+        line = [float(x) for x in line.split()]
+        position = bproc.math.change_coordinate_frame_of_point(line[:3], ["X", "-Z", "Y"])
+        rotation = bproc.math.change_coordinate_frame_of_point(line[3:6], ["X", "-Z", "Y"])
+        matrix_world = bproc.math.build_transformation_mat(position, bproc.camera.rotation_from_forward_vec(rotation))
+        bproc.camera.add_camera_pose(matrix_world)
+
+# makes Suncg objects emit light
+bproc.lighting.light_suncg_scene()
+
+# activate normal and depth rendering
+bproc.renderer.enable_depth_output(activate_antialiasing=False)
+bproc.material.add_alpha_channel_to_textures(blurry_edges=True)
+bproc.renderer.toggle_stereo(True)
+
+# render the whole pipeline
+data = bproc.renderer.render()
+
+# Apply stereo matching to each pair of images
+data["stereo-depth"], data["disparity"] = bproc.postprocessing.stereo_global_matching(data["colors"], disparity_filter=False)
+
+# write the data to a .hdf5 container
+bproc.writer.write_hdf5(args.output_dir, data)