From f9ccf18a1776820c56fd595ec1fa850498177f9c Mon Sep 17 00:00:00 2001 From: Hemal Shah Date: Thu, 19 Oct 2023 02:20:28 +0000 Subject: [PATCH] Isaac ROS 2.0.0 --- README.md | 371 +---- docs/bi3d-example.md | 31 - docs/tutorial-bi3d-isaac-sim.md | 52 - isaac_ros_bi3d/CMakeLists.txt | 14 +- isaac_ros_bi3d/config/bi3d_node.yaml | 156 ++- .../config/isaac_ros_bi3d_isaac_sim.rviz | 145 +- isaac_ros_bi3d/config/isaac_ros_bi3d_zed.rviz | 197 +++ isaac_ros_bi3d/config/zed.yaml | 163 +++ isaac_ros_bi3d/gxf/CMakeLists.txt | 94 ++ isaac_ros_bi3d/gxf/bi3d/3dv/src/Bi3D_detail.h | 340 ----- isaac_ros_bi3d/gxf/bi3d/CMakeLists.txt | 135 -- .../bi3d/bi3d_postprocessor/CMakeLists.txt | 51 - .../bi3d_message_splitter.cpp | 67 - .../bi3d_message_splitter.hpp | 50 - .../bi3d_postprocess/bi3d_postprocessor.cpp | 178 --- .../bi3d_postprocessor.cu.cpp | 54 - .../bi3d_postprocess/bi3d_postprocessor.hpp | 55 - .../bi3d_postprocessor_extension.cpp | 45 - .../gxf/bi3d/cvcore/include/cv/core/Array.h | 386 ------ .../gxf/bi3d/cvcore/include/cv/core/BBox.h | 142 -- .../gxf/bi3d/cvcore/include/cv/core/CVError.h | 116 -- .../bi3d/cvcore/include/cv/core/CameraModel.h | 292 ---- .../cvcore/include/cv/core/ComputeEngine.h | 43 - .../gxf/bi3d/cvcore/include/cv/core/Core.h | 35 - .../gxf/bi3d/cvcore/include/cv/core/Image.h | 893 ------------- .../cvcore/include/cv/core/Instrumentation.h | 65 - .../bi3d/cvcore/include/cv/core/MathTypes.h | 234 ---- .../gxf/bi3d/cvcore/include/cv/core/Memory.h | 135 -- .../cvcore/include/cv/core/ProfileUtils.h | 40 - .../gxf/bi3d/cvcore/include/cv/core/Tensor.h | 1189 ----------------- .../bi3d/cvcore/include/cv/core/TensorList.h | 37 - .../bi3d/cvcore/include/cv/core/TensorMap.h | 534 -------- .../gxf/bi3d/cvcore/include/cv/core/Traits.h | 478 ------- .../cvcore/include/cv/tensor_ops/BBoxUtils.h | 135 -- .../cvcore/include/cv/tensor_ops/DBScan.h | 91 -- .../cvcore/include/cv/tensor_ops/Errors.h | 49 - .../cvcore/include/cv/tensor_ops/IImageWarp.h | 63 - .../cv/tensor_ops/ITensorOperatorContext.h | 65 - .../cv/tensor_ops/ITensorOperatorStream.h | 251 ---- .../cvcore/include/cv/tensor_ops/ImageUtils.h | 1091 --------------- .../cvcore/include/cv/tensor_ops/NppUtils.cpp | 116 -- .../cvcore/include/cv/tensor_ops/NppUtils.h | 31 - .../include/cv/tensor_ops/OneEuroFilter.h | 82 -- .../include/cv/tensor_ops/TensorOperators.h | 48 - .../cvcore/include/cv/trtbackend/TRTBackend.h | 203 --- .../gxf/bi3d/cvcore/src/core/cvcore/Array.cpp | 145 -- .../gxf/bi3d/cvcore/src/core/cvcore/Dummy.cu | 0 .../bi3d/cvcore/src/core/cvcore/MathTypes.cpp | 244 ---- .../bi3d/cvcore/src/core/cvcore/Tensor.cpp | 270 ---- .../bi3d/cvcore/src/core/utility/CVError.cpp | 123 -- .../src/core/utility/Instrumentation.cpp | 95 -- .../bi3d/cvcore/src/core/utility/Memory.cpp | 124 -- .../cvcore/src/core/utility/ProfileUtils.cpp | 128 -- .../tensorrt/TensorRTInferencer.cpp | 275 ---- .../src/inferencer/tensorrt/TensorRTUtils.cpp | 65 - .../src/inferencer/triton/TritonUtils.cpp | 84 -- .../src/tensor_ops/ArithmeticOperations.cpp | 329 ----- .../bi3d/cvcore/src/tensor_ops/BBoxUtils.cpp | 173 --- .../src/tensor_ops/ColorConversions.cpp | 447 ------- .../gxf/bi3d/cvcore/src/tensor_ops/DBScan.cpp | 214 --- .../gxf/bi3d/cvcore/src/tensor_ops/Errors.cpp | 104 -- .../bi3d/cvcore/src/tensor_ops/Filters.cpp | 112 -- .../gxf/bi3d/cvcore/src/tensor_ops/Filters.h | 105 -- .../cvcore/src/tensor_ops/FusedOperations.cpp | 261 ---- .../src/tensor_ops/GeometryTransforms.cpp | 754 ----------- .../bi3d/cvcore/src/tensor_ops/NppUtils.cpp | 116 -- .../gxf/bi3d/cvcore/src/tensor_ops/NppUtils.h | 31 - .../cvcore/src/tensor_ops/OneEuroFilter.cpp | 288 ---- .../cvcore/src/tensor_ops/TensorOperators.cpp | 116 -- .../tensor_ops/vpi/VPIColorConvertImpl.cpp | 135 -- .../src/tensor_ops/vpi/VPIColorConvertImpl.h | 65 - .../src/tensor_ops/vpi/VPIEnumMapping.h | 196 --- .../cvcore/src/tensor_ops/vpi/VPIImageWarp.h | 37 - .../src/tensor_ops/vpi/VPIRemapImpl.cpp | 160 --- .../cvcore/src/tensor_ops/vpi/VPIRemapImpl.h | 82 -- .../src/tensor_ops/vpi/VPIResizeImpl.cpp | 139 -- .../cvcore/src/tensor_ops/vpi/VPIResizeImpl.h | 66 - .../src/tensor_ops/vpi/VPIStatusMapping.cpp | 122 -- .../src/tensor_ops/vpi/VPIStatusMapping.h | 38 - .../vpi/VPIStereoDisparityEstimatorImpl.cpp | 211 --- .../vpi/VPIStereoDisparityEstimatorImpl.h | 83 -- .../src/tensor_ops/vpi/VPITensorOperators.cpp | 710 ---------- .../src/tensor_ops/vpi/VPITensorOperators.h | 272 ---- .../bi3d/cvcore/src/trtbackend/TRTBackend.cpp | 634 --------- .../gxf/bi3d/extensions/bi3d/Bi3D.cpp | 476 ------- .../gxf/bi3d/extensions/bi3d/Bi3DRegistry.cpp | 27 - .../extensions/tensor_ops/CameraModel.cpp | 86 -- .../extensions/tensor_ops/CameraModel.hpp | 60 - .../tensor_ops/ConvertColorFormat.cpp | 214 --- .../tensor_ops/ConvertColorFormat.hpp | 51 - .../extensions/tensor_ops/CropAndResize.cpp | 161 --- .../extensions/tensor_ops/CropAndResize.hpp | 53 - .../bi3d/extensions/tensor_ops/Frame3D.cpp | 56 - .../bi3d/extensions/tensor_ops/Frame3D.hpp | 53 - .../extensions/tensor_ops/ImageAdapter.cpp | 78 -- .../extensions/tensor_ops/ImageAdapter.hpp | 101 -- .../bi3d/extensions/tensor_ops/ImageUtils.cpp | 175 --- .../bi3d/extensions/tensor_ops/ImageUtils.hpp | 65 - .../tensor_ops/InterleavedToPlanar.cpp | 146 -- .../tensor_ops/InterleavedToPlanar.hpp | 45 - .../bi3d/extensions/tensor_ops/Normalize.cpp | 183 --- .../bi3d/extensions/tensor_ops/Normalize.hpp | 47 - .../bi3d/extensions/tensor_ops/Reshape.cpp | 98 -- .../bi3d/extensions/tensor_ops/Reshape.hpp | 49 - .../gxf/bi3d/extensions/tensor_ops/Resize.cpp | 194 --- .../gxf/bi3d/extensions/tensor_ops/Resize.hpp | 55 - .../extensions/tensor_ops/TensorOperator.cpp | 235 ---- .../extensions/tensor_ops/TensorOperator.hpp | 95 -- .../bi3d/extensions/tensor_ops/TensorOps.cpp | 91 -- .../extensions/tensor_ops/TensorStream.cpp | 124 -- .../extensions/tensor_ops/TensorStream.hpp | 59 - .../bi3d/extensions/tensor_ops/Undistort.cpp | 285 ---- .../bi3d/extensions/tensor_ops/Undistort.hpp | 69 - .../detail/ImageAdapterTensorImpl.cpp | 105 -- .../detail/ImageAdapterTensorImpl.hpp | 105 -- .../detail/ImageAdapterVideoBufferImpl.cpp | 88 -- .../detail/ImageAdapterVideoBufferImpl.hpp | 294 ---- .../gxf/bi3d_postprocessor/CMakeLists.txt | 60 - .../bi3d_message_splitter.cpp | 67 - .../bi3d_message_splitter.hpp | 50 - .../bi3d_postprocess/bi3d_postprocessor.cpp | 178 --- .../bi3d_postprocessor.cu.cpp | 54 - .../bi3d_postprocessor_extension.cpp | 45 - .../bi3d_postprocessor_utils.hpp | 87 -- isaac_ros_bi3d/gxf/extensions/bi3d/bi3d.cpp | 40 + .../bi3d/components/bi3d_depthview.cpp | 200 +++ .../bi3d/components/bi3d_depthview.hpp | 63 + .../bi3d/components/bi3d_inference.cpp | 551 ++++++++ .../bi3d/components/bi3d_inference.hpp} | 50 +- .../bi3d/components/bi3d_postprocessor.cpp | 128 ++ .../bi3d/components/bi3d_postprocessor.cu | 42 + .../components}/bi3d_postprocessor.cu.hpp | 20 +- .../bi3d/components}/bi3d_postprocessor.hpp | 33 +- .../components}/bi3d_postprocessor_utils.hpp | 74 +- .../bi3d/inference}/Bi3D.cpp | 557 ++++---- .../bi3d => extensions/bi3d/inference}/Bi3D.h | 149 ++- .../bi3d/inference}/Bi3DPostProcessor.cpp | 113 +- .../bi3d/inference}/Bi3DPreProcessor.cpp | 111 +- .../bi3d/inference}/Bi3D_detail.cpp | 85 +- .../extensions/bi3d/inference/Bi3D_detail.hpp | 343 +++++ .../bi3d/inference}/Model.h | 24 +- .../dnn_inferencer}/inferencer/Errors.cpp | 45 +- .../dnn_inferencer}/inferencer/Errors.h | 28 +- .../inferencer/IInferenceBackend.h | 97 +- .../dnn_inferencer}/inferencer/Inferencer.cpp | 81 +- .../dnn_inferencer}/inferencer/Inferencer.h | 49 +- .../inferencer/TensorRTInferencer.cpp | 387 ++++++ .../inferencer}/TensorRTInferencer.h | 62 +- .../inferencer/TensorRTUtils.cpp | 57 + .../inferencer}/TensorRTUtils.h | 24 +- .../inferencer/TritionUtils.cpp | 64 + .../inferencer}/TritonGrpcInferencer.cpp | 203 ++- .../inferencer}/TritonGrpcInferencer.h | 51 +- .../dnn_inferencer/inferencer}/TritonUtils.h | 20 +- .../gxf_helpers/common_expected_macro.hpp | 305 +++++ .../gxf/gems/gxf_helpers/element_type.hpp | 55 + .../gxf_helpers/expected_macro.hpp} | 32 +- .../gxf/gems/gxf_helpers/tensor.hpp | 137 ++ isaac_ros_bi3d/gxf/gems/hash/hash_file.cpp | 71 + .../hash/hash_file.hpp} | 14 +- isaac_ros_bi3d/gxf/gems/hash/sha256.cpp | 237 ++++ isaac_ros_bi3d/gxf/gems/hash/sha256.hpp | 80 ++ .../gxf/gems/video_buffer/allocator.hpp | 276 ++++ .../include/isaac_ros_bi3d/bi3d_node.hpp | 6 +- isaac_ros_bi3d/isaac_ros_bi3d/__init__.py | 16 + .../launch/isaac_ros_bi3d.launch.py | 16 +- .../launch/isaac_ros_bi3d_isaac_sim.launch.py | 54 +- .../launch/isaac_ros_bi3d_realsense.launch.py | 30 +- .../launch/isaac_ros_bi3d_zed.launch.py | 190 +++ isaac_ros_bi3d/package.xml | 5 +- .../scripts/isaac_ros_bi3d_visualizer.py | 2 +- isaac_ros_bi3d/src/bi3d_node.cpp | 108 +- isaac_ros_bi3d/test/camera_info.json | 51 + isaac_ros_bi3d/test/isaac_ros_bi3d_test.py | 27 +- .../CMakeLists.txt | 8 +- .../package.xml | 2 +- resources/Isaac_sim_enable_stereo.png | 3 - resources/Isaac_sim_play.png | 3 - resources/Isaac_sim_rviz.png | 3 - resources/Isaac_sim_set_stereo_offset.png | 3 - resources/Isaac_sim_tutorial.gif | 3 - resources/Rviz.png | 3 - resources/Visualizer_isaac_sim.png | 3 - resources/bi3d_left.png | 3 - resources/bi3d_output_1mps.png | 3 - resources/bi3d_output_2mps.png | 3 - resources/bi3d_output_3mps.png | 3 - resources/bi3d_visualizer_output.gif | 3 - resources/bi3d_visualizer_output.png | 3 - resources/disparity_equation.png | 3 - resources/freespace_example_real.png | 3 - resources/freespace_example_segmented.png | 3 - resources/isaac_ros_bi3d_nodegraph.png | 3 - resources/isaac_ros_bi3d_real_opt.gif | 3 - resources/quickstart_disparity.png | 3 - resources/quickstart_rgb.png | 3 - resources/realsense_camera_position.jpg | 3 - resources/realsense_example.gif | 3 - resources/realsense_example.png | 3 - resources/safety_zones.png | 3 - resources/visualizer_realsense.png | 3 - resources/visualizer_realsense_mono_pair.png | 3 - 202 files changed, 5189 insertions(+), 21521 deletions(-) delete mode 100644 docs/bi3d-example.md delete mode 100644 docs/tutorial-bi3d-isaac-sim.md create mode 100644 isaac_ros_bi3d/config/isaac_ros_bi3d_zed.rviz create mode 100644 isaac_ros_bi3d/config/zed.yaml create mode 100644 isaac_ros_bi3d/gxf/CMakeLists.txt delete mode 100644 isaac_ros_bi3d/gxf/bi3d/3dv/src/Bi3D_detail.h delete mode 100644 isaac_ros_bi3d/gxf/bi3d/CMakeLists.txt delete mode 100644 isaac_ros_bi3d/gxf/bi3d/bi3d_postprocessor/CMakeLists.txt delete mode 100644 isaac_ros_bi3d/gxf/bi3d/bi3d_postprocessor/bi3d_postprocess/bi3d_message_splitter.cpp delete mode 100644 isaac_ros_bi3d/gxf/bi3d/bi3d_postprocessor/bi3d_postprocess/bi3d_message_splitter.hpp delete mode 100644 isaac_ros_bi3d/gxf/bi3d/bi3d_postprocessor/bi3d_postprocess/bi3d_postprocessor.cpp delete mode 100644 isaac_ros_bi3d/gxf/bi3d/bi3d_postprocessor/bi3d_postprocess/bi3d_postprocessor.cu.cpp delete mode 100644 isaac_ros_bi3d/gxf/bi3d/bi3d_postprocessor/bi3d_postprocess/bi3d_postprocessor.hpp delete mode 100644 isaac_ros_bi3d/gxf/bi3d/bi3d_postprocessor/bi3d_postprocessor_extension.cpp delete mode 100644 isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/core/Array.h delete mode 100644 isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/core/BBox.h delete mode 100644 isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/core/CVError.h delete mode 100644 isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/core/CameraModel.h delete mode 100644 isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/core/ComputeEngine.h delete mode 100644 isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/core/Core.h delete mode 100644 isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/core/Image.h delete mode 100644 isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/core/Instrumentation.h delete mode 100644 isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/core/MathTypes.h delete mode 100644 isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/core/Memory.h delete mode 100644 isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/core/ProfileUtils.h delete mode 100644 isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/core/Tensor.h delete mode 100644 isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/core/TensorList.h delete mode 100644 isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/core/TensorMap.h delete mode 100644 isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/core/Traits.h delete mode 100644 isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/tensor_ops/BBoxUtils.h delete mode 100644 isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/tensor_ops/DBScan.h delete mode 100644 isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/tensor_ops/Errors.h delete mode 100644 isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/tensor_ops/IImageWarp.h delete mode 100644 isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/tensor_ops/ITensorOperatorContext.h delete mode 100644 isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/tensor_ops/ITensorOperatorStream.h delete mode 100644 isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/tensor_ops/ImageUtils.h delete mode 100644 isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/tensor_ops/NppUtils.cpp delete mode 100644 isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/tensor_ops/NppUtils.h delete mode 100644 isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/tensor_ops/OneEuroFilter.h delete mode 100644 isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/tensor_ops/TensorOperators.h delete mode 100644 isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/trtbackend/TRTBackend.h delete mode 100644 isaac_ros_bi3d/gxf/bi3d/cvcore/src/core/cvcore/Array.cpp delete mode 100644 isaac_ros_bi3d/gxf/bi3d/cvcore/src/core/cvcore/Dummy.cu delete mode 100644 isaac_ros_bi3d/gxf/bi3d/cvcore/src/core/cvcore/MathTypes.cpp delete mode 100644 isaac_ros_bi3d/gxf/bi3d/cvcore/src/core/cvcore/Tensor.cpp delete mode 100644 isaac_ros_bi3d/gxf/bi3d/cvcore/src/core/utility/CVError.cpp delete mode 100644 isaac_ros_bi3d/gxf/bi3d/cvcore/src/core/utility/Instrumentation.cpp delete mode 100644 isaac_ros_bi3d/gxf/bi3d/cvcore/src/core/utility/Memory.cpp delete mode 100644 isaac_ros_bi3d/gxf/bi3d/cvcore/src/core/utility/ProfileUtils.cpp delete mode 100644 isaac_ros_bi3d/gxf/bi3d/cvcore/src/inferencer/tensorrt/TensorRTInferencer.cpp delete mode 100644 isaac_ros_bi3d/gxf/bi3d/cvcore/src/inferencer/tensorrt/TensorRTUtils.cpp delete mode 100644 isaac_ros_bi3d/gxf/bi3d/cvcore/src/inferencer/triton/TritonUtils.cpp delete mode 100644 isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/ArithmeticOperations.cpp delete mode 100644 isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/BBoxUtils.cpp delete mode 100644 isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/ColorConversions.cpp delete mode 100644 isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/DBScan.cpp delete mode 100644 isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/Errors.cpp delete mode 100644 isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/Filters.cpp delete mode 100644 isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/Filters.h delete mode 100644 isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/FusedOperations.cpp delete mode 100644 isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/GeometryTransforms.cpp delete mode 100644 isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/NppUtils.cpp delete mode 100644 isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/NppUtils.h delete mode 100644 isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/OneEuroFilter.cpp delete mode 100644 isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/TensorOperators.cpp delete mode 100644 isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/vpi/VPIColorConvertImpl.cpp delete mode 100644 isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/vpi/VPIColorConvertImpl.h delete mode 100644 isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/vpi/VPIEnumMapping.h delete mode 100644 isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/vpi/VPIImageWarp.h delete mode 100644 isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/vpi/VPIRemapImpl.cpp delete mode 100644 isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/vpi/VPIRemapImpl.h delete mode 100644 isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/vpi/VPIResizeImpl.cpp delete mode 100644 isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/vpi/VPIResizeImpl.h delete mode 100644 isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/vpi/VPIStatusMapping.cpp delete mode 100644 isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/vpi/VPIStatusMapping.h delete mode 100644 isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/vpi/VPIStereoDisparityEstimatorImpl.cpp delete mode 100644 isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/vpi/VPIStereoDisparityEstimatorImpl.h delete mode 100644 isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/vpi/VPITensorOperators.cpp delete mode 100644 isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/vpi/VPITensorOperators.h delete mode 100644 isaac_ros_bi3d/gxf/bi3d/cvcore/src/trtbackend/TRTBackend.cpp delete mode 100644 isaac_ros_bi3d/gxf/bi3d/extensions/bi3d/Bi3D.cpp delete mode 100644 isaac_ros_bi3d/gxf/bi3d/extensions/bi3d/Bi3DRegistry.cpp delete mode 100644 isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/CameraModel.cpp delete mode 100644 isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/CameraModel.hpp delete mode 100644 isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/ConvertColorFormat.cpp delete mode 100644 isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/ConvertColorFormat.hpp delete mode 100644 isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/CropAndResize.cpp delete mode 100644 isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/CropAndResize.hpp delete mode 100644 isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/Frame3D.cpp delete mode 100644 isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/Frame3D.hpp delete mode 100644 isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/ImageAdapter.cpp delete mode 100644 isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/ImageAdapter.hpp delete mode 100644 isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/ImageUtils.cpp delete mode 100644 isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/ImageUtils.hpp delete mode 100644 isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/InterleavedToPlanar.cpp delete mode 100644 isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/InterleavedToPlanar.hpp delete mode 100644 isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/Normalize.cpp delete mode 100644 isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/Normalize.hpp delete mode 100644 isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/Reshape.cpp delete mode 100644 isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/Reshape.hpp delete mode 100644 isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/Resize.cpp delete mode 100644 isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/Resize.hpp delete mode 100644 isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/TensorOperator.cpp delete mode 100644 isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/TensorOperator.hpp delete mode 100644 isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/TensorOps.cpp delete mode 100644 isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/TensorStream.cpp delete mode 100644 isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/TensorStream.hpp delete mode 100644 isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/Undistort.cpp delete mode 100644 isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/Undistort.hpp delete mode 100644 isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/detail/ImageAdapterTensorImpl.cpp delete mode 100644 isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/detail/ImageAdapterTensorImpl.hpp delete mode 100644 isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/detail/ImageAdapterVideoBufferImpl.cpp delete mode 100644 isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/detail/ImageAdapterVideoBufferImpl.hpp delete mode 100644 isaac_ros_bi3d/gxf/bi3d_postprocessor/CMakeLists.txt delete mode 100644 isaac_ros_bi3d/gxf/bi3d_postprocessor/bi3d_postprocess/bi3d_message_splitter.cpp delete mode 100644 isaac_ros_bi3d/gxf/bi3d_postprocessor/bi3d_postprocess/bi3d_message_splitter.hpp delete mode 100644 isaac_ros_bi3d/gxf/bi3d_postprocessor/bi3d_postprocess/bi3d_postprocessor.cpp delete mode 100644 isaac_ros_bi3d/gxf/bi3d_postprocessor/bi3d_postprocess/bi3d_postprocessor.cu.cpp delete mode 100644 isaac_ros_bi3d/gxf/bi3d_postprocessor/bi3d_postprocessor_extension.cpp delete mode 100644 isaac_ros_bi3d/gxf/bi3d_postprocessor/bi3d_postprocessor_utils.hpp create mode 100644 isaac_ros_bi3d/gxf/extensions/bi3d/bi3d.cpp create mode 100644 isaac_ros_bi3d/gxf/extensions/bi3d/components/bi3d_depthview.cpp create mode 100644 isaac_ros_bi3d/gxf/extensions/bi3d/components/bi3d_depthview.hpp create mode 100644 isaac_ros_bi3d/gxf/extensions/bi3d/components/bi3d_inference.cpp rename isaac_ros_bi3d/gxf/{bi3d/extensions/bi3d/Bi3D.hpp => extensions/bi3d/components/bi3d_inference.hpp} (82%) create mode 100644 isaac_ros_bi3d/gxf/extensions/bi3d/components/bi3d_postprocessor.cpp create mode 100644 isaac_ros_bi3d/gxf/extensions/bi3d/components/bi3d_postprocessor.cu rename isaac_ros_bi3d/gxf/{bi3d_postprocessor/bi3d_postprocess => extensions/bi3d/components}/bi3d_postprocessor.cu.hpp (69%) rename isaac_ros_bi3d/gxf/{bi3d_postprocessor/bi3d_postprocess => extensions/bi3d/components}/bi3d_postprocessor.hpp (67%) rename isaac_ros_bi3d/gxf/{bi3d/bi3d_postprocessor => extensions/bi3d/components}/bi3d_postprocessor_utils.hpp (52%) rename isaac_ros_bi3d/gxf/{bi3d/3dv/src => extensions/bi3d/inference}/Bi3D.cpp (50%) rename isaac_ros_bi3d/gxf/{bi3d/3dv/include/cv/bi3d => extensions/bi3d/inference}/Bi3D.h (58%) rename isaac_ros_bi3d/gxf/{bi3d/3dv/src => extensions/bi3d/inference}/Bi3DPostProcessor.cpp (59%) rename isaac_ros_bi3d/gxf/{bi3d/3dv/src => extensions/bi3d/inference}/Bi3DPreProcessor.cpp (54%) rename isaac_ros_bi3d/gxf/{bi3d/3dv/src => extensions/bi3d/inference}/Bi3D_detail.cpp (79%) create mode 100644 isaac_ros_bi3d/gxf/extensions/bi3d/inference/Bi3D_detail.hpp rename isaac_ros_bi3d/gxf/{bi3d/cvcore/include/cv/core => extensions/bi3d/inference}/Model.h (79%) rename isaac_ros_bi3d/gxf/{bi3d/cvcore/src => gems/dnn_inferencer}/inferencer/Errors.cpp (86%) rename isaac_ros_bi3d/gxf/{bi3d/cvcore/include/cv => gems/dnn_inferencer}/inferencer/Errors.h (77%) rename isaac_ros_bi3d/gxf/{bi3d/cvcore/include/cv => gems/dnn_inferencer}/inferencer/IInferenceBackend.h (71%) rename isaac_ros_bi3d/gxf/{bi3d/cvcore/src => gems/dnn_inferencer}/inferencer/Inferencer.cpp (69%) rename isaac_ros_bi3d/gxf/{bi3d/cvcore/include/cv => gems/dnn_inferencer}/inferencer/Inferencer.h (69%) create mode 100644 isaac_ros_bi3d/gxf/gems/dnn_inferencer/inferencer/TensorRTInferencer.cpp rename isaac_ros_bi3d/gxf/{bi3d/cvcore/src/inferencer/tensorrt => gems/dnn_inferencer/inferencer}/TensorRTInferencer.h (54%) create mode 100644 isaac_ros_bi3d/gxf/gems/dnn_inferencer/inferencer/TensorRTUtils.cpp rename isaac_ros_bi3d/gxf/{bi3d/cvcore/src/inferencer/tensorrt => gems/dnn_inferencer/inferencer}/TensorRTUtils.h (69%) create mode 100644 isaac_ros_bi3d/gxf/gems/dnn_inferencer/inferencer/TritionUtils.cpp rename isaac_ros_bi3d/gxf/{bi3d/cvcore/src/inferencer/triton => gems/dnn_inferencer/inferencer}/TritonGrpcInferencer.cpp (74%) rename isaac_ros_bi3d/gxf/{bi3d/cvcore/src/inferencer/triton => gems/dnn_inferencer/inferencer}/TritonGrpcInferencer.h (60%) rename isaac_ros_bi3d/gxf/{bi3d/cvcore/src/inferencer/triton => gems/dnn_inferencer/inferencer}/TritonUtils.h (81%) create mode 100644 isaac_ros_bi3d/gxf/gems/gxf_helpers/common_expected_macro.hpp create mode 100644 isaac_ros_bi3d/gxf/gems/gxf_helpers/element_type.hpp rename isaac_ros_bi3d/gxf/{bi3d/bi3d_postprocessor/bi3d_postprocess/bi3d_postprocessor.cu.hpp => gems/gxf_helpers/expected_macro.hpp} (52%) create mode 100644 isaac_ros_bi3d/gxf/gems/gxf_helpers/tensor.hpp create mode 100644 isaac_ros_bi3d/gxf/gems/hash/hash_file.cpp rename isaac_ros_bi3d/gxf/{bi3d/cvcore/src/tensor_ops/IImageWarp.cpp => gems/hash/hash_file.hpp} (68%) create mode 100644 isaac_ros_bi3d/gxf/gems/hash/sha256.cpp create mode 100644 isaac_ros_bi3d/gxf/gems/hash/sha256.hpp create mode 100644 isaac_ros_bi3d/gxf/gems/video_buffer/allocator.hpp create mode 100644 isaac_ros_bi3d/launch/isaac_ros_bi3d_zed.launch.py create mode 100644 isaac_ros_bi3d/test/camera_info.json delete mode 100644 resources/Isaac_sim_enable_stereo.png delete mode 100644 resources/Isaac_sim_play.png delete mode 100644 resources/Isaac_sim_rviz.png delete mode 100644 resources/Isaac_sim_set_stereo_offset.png delete mode 100644 resources/Isaac_sim_tutorial.gif delete mode 100644 resources/Rviz.png delete mode 100644 resources/Visualizer_isaac_sim.png delete mode 100644 resources/bi3d_left.png delete mode 100644 resources/bi3d_output_1mps.png delete mode 100644 resources/bi3d_output_2mps.png delete mode 100644 resources/bi3d_output_3mps.png delete mode 100644 resources/bi3d_visualizer_output.gif delete mode 100644 resources/bi3d_visualizer_output.png delete mode 100644 resources/disparity_equation.png delete mode 100644 resources/freespace_example_real.png delete mode 100644 resources/freespace_example_segmented.png delete mode 100644 resources/isaac_ros_bi3d_nodegraph.png delete mode 100644 resources/isaac_ros_bi3d_real_opt.gif delete mode 100644 resources/quickstart_disparity.png delete mode 100644 resources/quickstart_rgb.png delete mode 100644 resources/realsense_camera_position.jpg delete mode 100644 resources/realsense_example.gif delete mode 100644 resources/realsense_example.png delete mode 100644 resources/safety_zones.png delete mode 100644 resources/visualizer_realsense.png delete mode 100644 resources/visualizer_realsense_mono_pair.png diff --git a/README.md b/README.md index 1e6f73e..75cb64d 100644 --- a/README.md +++ b/README.md @@ -1,334 +1,81 @@ -# Isaac ROS Proximity Segmentation +# Isaac ROS Depth Segmentation -
Isaac ROS Proximity Segmentation Sample Output
+Hardware-accelerated packages for depth segmentation. ---- -## Webinar Available -Learn how to use this package by watching our on-demand webinar: [Using ML Models in ROS 2 to Robustly Estimate Distance to Obstacles](https://gateway.on24.com/wcc/experience/elitenvidiabrill/1407606/3998202/isaac-ros-webinar-series) +
image
--- -## Overview - -This repository provides NVIDIA hardware-accelerated packages for proximity segmentation. The `isaac_ros_bi3d` package uses the optimized [Bi3D DNN model](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/isaac/models/bi3d_proximity_segmentation) to perform stereo-depth estimation via binary classification, which is used for proximity segmentation. Proximity segmentation can be used to determine whether an obstacle is within a proximity field and to avoid collisions with obstacles during navigation. +## Webinar Available -
graph of nodes with Bi3D
+Learn how to use this package by watching our on-demand webinar: [Using ML Models in ROS 2 to Robustly Estimate Distance to Obstacles](https://gateway.on24.com/wcc/experience/elitenvidiabrill/1407606/3998202/isaac-ros-webinar-series) -[Bi3D](https://arxiv.org/abs/2005.07274) is used in a graph of nodes to provide proximity segmentation from a time-synchronized input left and right stereo image pair. Images to Bi3D need to be rectified and resized to the appropriate input resolution. The aspect ratio of the image needs to be maintained; hence, a crop and resize may be required to maintain the input aspect ratio. The graph for DNN encode, to DNN inference, to DNN decode is part of the Bi3D node. Inference is performed using TensorRT, as the Bi3D DNN model is designed to use optimizations supported by TensorRT. +--- -Compared to other stereo disparity functions, proximity segmentation provides a prediction of whether an obstacle is within a proximity field, as opposed to continuous depth, while simultaneously predicting freespace from the ground plane, which other functions typically do not provide. Also unlike other stereo disparity functions in Isaac ROS, proximity segmentation runs on NVIDIA DLA (deep learning accelerator), which is separate and independent from the GPU. For more information on disparity, refer to [this page](https://en.wikipedia.org/wiki/Binocular_disparity). +### Overview + +[Isaac ROS Depth Segmentation](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_depth_segmentation) provides NVIDIA hardware-accelerated packages for +depth segmentation. The `isaac_ros_bi3d` package uses the +optimized [Bi3D DNN +model](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/isaac/models/bi3d_proximity_segmentation) +to perform stereo-depth estimation via binary classification, which is +used for depth segmentation. Depth segmentation can be used to +determine whether an obstacle is within a proximity field and to avoid +collisions with obstacles during navigation. + +
image
+ +[Bi3D](https://arxiv.org/abs/2005.07274) is used in a graph of nodes +to provide depth segmentation from a time-synchronized input left +and right stereo image pair. Images to Bi3D need to be rectified and +resized to the appropriate input resolution. The aspect ratio of the +image needs to be maintained; hence, a crop and resize may be required +to maintain the input aspect ratio. The graph for DNN encode, to DNN +inference, to DNN decode is part of the Bi3D node. Inference is +performed using TensorRT, as the Bi3D DNN model is designed to use +optimizations supported by TensorRT. + +Compared to other stereo disparity functions, depth segmentation +provides a prediction of whether an obstacle is within a proximity +field, as opposed to continuous depth, while simultaneously predicting +freespace from the ground plane, which other functions typically do not +provide. Also unlike other stereo disparity functions in Isaac ROS, +depth segmentation runs on NVIDIA DLA (deep learning accelerator), +which is separate and independent from the GPU. For more information on +disparity, refer to [this +page](https://en.wikipedia.org/wiki/Binocular_disparity). + +> [!Note] +> This DNN is optimized for and evaluated with RGB global shutter camera images, +> and accuracy may vary on monochrome images. ### Isaac ROS NITROS Acceleration This package is powered by [NVIDIA Isaac Transport for ROS (NITROS)](https://developer.nvidia.com/blog/improve-perception-performance-for-ros-2-applications-with-nvidia-isaac-transport-for-ros/), which leverages type adaptation and negotiation to optimize message formats and dramatically accelerate communication between participating nodes. -## Performance - -The following table summarizes the per-platform performance statistics of sample graphs that use this package, with links included to the full benchmark output. These benchmark configurations are taken from the [Isaac ROS Benchmark](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark#list-of-isaac-ros-benchmarks) collection, based on the [`ros2_benchmark`](https://github.com/NVIDIA-ISAAC-ROS/ros2_benchmark) framework. - -| Sample Graph | Input Size | AGX Orin | Orin NX | x86_64 w/ RTX 4060 Ti | -| ---------------------------------------------------------------------------------------------------------------------------------------- | ---------- | ---------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------- | -| [Proximity Segmentation Node](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/scripts//isaac_ros_bi3d_node.py) | 576p | [49.3 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_bi3d_node-agx_orin.json)
32 ms | [24.2 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_bi3d_node-orin_nx.json)
65 ms | [159 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_bi3d_node-nuc_4060ti.json)
23 ms | - -## Table of Contents - -- [Isaac ROS Proximity Segmentation](#isaac-ros-proximity-segmentation) - - [Webinar Available](#webinar-available) - - [Overview](#overview) - - [Isaac ROS NITROS Acceleration](#isaac-ros-nitros-acceleration) - - [Performance](#performance) - - [Table of Contents](#table-of-contents) - - [Latest Update](#latest-update) - - [Supported Platforms](#supported-platforms) - - [Docker](#docker) - - [Quickstart](#quickstart) - - [Next Steps](#next-steps) - - [Try More Examples](#try-more-examples) - - [Use Different Models](#use-different-models) - - [Customize your Dev Environment](#customize-your-dev-environment) - - [Model Preparation](#model-preparation) - - [Download Pre-trained Models (.onnx) from NGC](#download-pre-trained-models-onnx-from-ngc) - - [Convert the Pre-trained Models (.onnx) to TensorRT Engine Plans](#convert-the-pre-trained-models-onnx-to-tensorrt-engine-plans) - - [Generating Engine Plans for Jetson](#generating-engine-plans-for-jetson) - - [Generating Engine Plans for x86\_64](#generating-engine-plans-for-x86_64) - - [Package Reference](#package-reference) - - [`isaac_ros_bi3d`](#isaac_ros_bi3d) - - [Bi3D Overview](#bi3d-overview) - - [Usage](#usage) - - [Interpreting the Output](#interpreting-the-output) - - [ROS Parameters](#ros-parameters) - - [ROS Topics Subscribed](#ros-topics-subscribed) - - [ROS Topics Published](#ros-topics-published) - - [Troubleshooting](#troubleshooting) - - [Isaac ROS Troubleshooting](#isaac-ros-troubleshooting) - - [DNN and Triton Troubleshooting](#dnn-and-triton-troubleshooting) - - [Updates](#updates) - -## Latest Update - -Update 2023-05-25: Performance improvements. - -## Supported Platforms - -This package is designed and tested to be compatible with ROS 2 Humble running on [Jetson](https://developer.nvidia.com/embedded-computing) or an x86_64 system with an NVIDIA GPU. - -> **Note**: Versions of ROS 2 earlier than Humble are **not** supported. This package depends on specific ROS 2 implementation features that were only introduced beginning with the Humble release. - -| Platform | Hardware | Software | Notes | -| -------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Jetson | [Jetson Orin](https://www.nvidia.com/en-us/autonomous-machines/embedded-systems/jetson-orin/)
[Jetson Xavier](https://www.nvidia.com/en-us/autonomous-machines/embedded-systems/jetson-agx-xavier/) | [JetPack 5.1.1](https://developer.nvidia.com/embedded/jetpack) | For best performance, ensure that the [power settings](https://docs.nvidia.com/jetson/archives/r34.1/DeveloperGuide/text/SD/PlatformPowerAndPerformance.html) are configured appropriately. | -| x86_64 | NVIDIA GPU | [Ubuntu 20.04+](https://releases.ubuntu.com/20.04/)
[CUDA 11.8+](https://developer.nvidia.com/cuda-downloads) | - -### Docker - -To simplify development, we strongly recommend leveraging the Isaac ROS Dev Docker images by following [these steps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_common/blob/main/docs/dev-env-setup.md). This will streamline your development environment setup with the correct versions of dependencies on both Jetson and x86_64 platforms. - -> **Note**: All Isaac ROS quick start guides, tutorials, and examples have been designed with the Isaac ROS Docker images as a prerequisite. - -## Quickstart - -1. Set up your development environment by following the instructions [here](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_common/blob/main/docs/dev-env-setup.md). - -2. Clone this repository and its dependencies under `~/workspaces/isaac_ros-dev/src`. - - ```bash - cd ~/workspaces/isaac_ros-dev/src && - git clone https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_common && - git clone https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_nitros && - git clone https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_image_pipeline && - git clone https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_proximity_segmentation - ``` - -3. Pull down a rosbag of sample data: - - ```bash - cd ~/workspaces/isaac_ros-dev/src/isaac_ros_proximity_segmentation && - git lfs pull -X "" -I "resources/rosbags/bi3dnode_rosbag" - ``` - -4. Launch the Docker container using the `run_dev.sh` script: - - ```bash - cd ~/workspaces/isaac_ros-dev/src/isaac_ros_common && - ./scripts/run_dev.sh - ``` - -5. Download model files for Bi3D (refer to the [Model Preparation](#model-preparation) section for more information): - - ```bash - mkdir -p /tmp/models/bi3d && - cd /tmp/models/bi3d && - wget 'https://api.ngc.nvidia.com/v2/models/nvidia/isaac/bi3d_proximity_segmentation/versions/2.0.0/files/featnet.onnx' && - wget 'https://api.ngc.nvidia.com/v2/models/nvidia/isaac/bi3d_proximity_segmentation/versions/2.0.0/files/segnet.onnx' - ``` - -6. Convert the `.onnx` model files to TensorRT engine plan files (refer to the [Model Preparation](#model-preparation) section for more information): - - If using Jetson (Generate engine plans with DLA support enabled): - - ```bash - /usr/src/tensorrt/bin/trtexec --saveEngine=/tmp/models/bi3d/bi3dnet_featnet.plan \ - --onnx=/tmp/models/bi3d/featnet.onnx \ - --int8 --useDLACore=0 --allowGPUFallback && - /usr/src/tensorrt/bin/trtexec --saveEngine=/tmp/models/bi3d/bi3dnet_segnet.plan \ - --onnx=/tmp/models/bi3d/segnet.onnx \ - --int8 --useDLACore=0 --allowGPUFallback - ``` - - If using x86_64: - - ```bash - /usr/src/tensorrt/bin/trtexec --saveEngine=/tmp/models/bi3d/bi3dnet_featnet.plan \ - --onnx=/tmp/models/bi3d/featnet.onnx --int8 && - /usr/src/tensorrt/bin/trtexec --saveEngine=/tmp/models/bi3d/bi3dnet_segnet.plan \ - --onnx=/tmp/models/bi3d/segnet.onnx --int8 - ``` - - > **Note**: The engine plans generated using the x86_64 commands will also work on Jetson, but performance will be reduced. - -7. Build and source the workspace: - - ```bash - cd /workspaces/isaac_ros-dev && - colcon build --symlink-install && - source install/setup.bash - ``` - -8. (Optional) Run tests to verify complete and correct installation: - - ```bash - colcon test --executor sequential - ``` - -9. Run the launch file to spin up a demo of this package: - - ```bash - ros2 launch isaac_ros_bi3d isaac_ros_bi3d.launch.py featnet_engine_file_path:=/tmp/models/bi3d/bi3dnet_featnet.plan \ - segnet_engine_file_path:=/tmp/models/bi3d/bi3dnet_segnet.plan \ - max_disparity_values:=10 - ``` - -10. Open a **second** terminal inside the Docker container: - - ```bash - cd ~/workspaces/isaac_ros-dev/src/isaac_ros_common && \ - ./scripts/run_dev.sh - ``` - -11. Play the rosbag file to simulate image streams from the cameras: - - ```bash - ros2 bag play --loop src/isaac_ros_proximity_segmentation/resources/rosbags/bi3dnode_rosbag - ``` +### Performance -12. Open two **new** terminals inside the Docker container for visualization: +| Sample Graph

| Input Size

| AGX Orin

| Orin NX

| x86_64 w/ RTX 4060 Ti

| +|-------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------| +| [Depth Segmentation Node](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/scripts/isaac_ros_bi3d_node.py)



| 576p



| [47.7 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_bi3d_node-agx_orin.json)


43 ms

| [30.0 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_bi3d_node-orin_nx.json)


98 ms

| [89.9 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_bi3d_node-nuc_4060ti.json)


28 ms

| - ```bash - cd ~/workspaces/isaac_ros-dev/src/isaac_ros_common && \ - ./scripts/run_dev.sh && source install/setup.bash - ``` - -13. Visualize the output. - - Start disparity visualizer: - - ```bash - ros2 run isaac_ros_bi3d isaac_ros_bi3d_visualizer.py --max_disparity_value 10 - ``` - Start image visualizer: - - ```bash - ros2 run image_view image_view --ros-args -r image:=rgb_left - ``` -
- RViz Output - RViz Output -
- -## Next Steps - -### Try More Examples - -To continue your exploration, check out the following suggested examples: - -| Example | Dependencies | -| ----------------------------------------------------------------------------- | ------------ | -| [Zone detection for an autonomous mobile robot (AMR)](./docs/bi3d-example.md) | -- | -| [Tutorial for Bi3D with Isaac Sim](./docs/tutorial-bi3d-isaac-sim.md) | -- | - -### Use Different Models - -Click [here](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_dnn_inference/blob/main/docs/model-preparation.md) for more information about how to use NGC models. - -### Customize your Dev Environment - -To customize your development environment, reference [this guide](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_common/blob/main/docs/modify-dockerfile.md). - -## Model Preparation - -### Download Pre-trained Models (.onnx) from NGC - -The following steps show how to download pretrained Bi3D DNN inference models. - -1. The following model files must be downloaded to perform Bi3D inference. From **File Browser** on the **Bi3D** [page](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/isaac/models/bi3d_proximity_segmentation), select the following `.onnx` model files in the **FILE** list and copy the `wget` command by clicking **...** in the **ACTIONS** column: - - `featnet.onnx` - - `segnet.onnx` - -2. Run each of the copied commands in a terminal to download the ONNX model file, as shown in the example below: - - ```bash - wget 'https://api.ngc.nvidia.com/v2/models/nvidia/isaac/bi3d_proximity_segmentation/versions/2.0.0/files/featnet.onnx' && - wget 'https://api.ngc.nvidia.com/v2/models/nvidia/isaac/bi3d_proximity_segmentation/versions/2.0.0/files/segnet.onnx' - ``` - -- Bi3D Featnet is a network that extracts features from stereo images. - -- Bi3D Segnet is an encoder-decoder segmentation network that generates a binary segmentation confidence map. - -### Convert the Pre-trained Models (.onnx) to TensorRT Engine Plans - -`trtexec` is used to convert pre-trained models (`.onnx`) to the TensorRT engine plan and is included in the Isaac ROS docker container under `/usr/src/tensorrt/bin/trtexec`. - -> **Tip**: Use `/usr/src/tensorrt/bin/trtexec -h` for more information on using the tool. - -#### Generating Engine Plans for Jetson - - ```bash - /usr/src/tensorrt/bin/trtexec --onnx= --saveEngine= --useDLACore= --int8 --allowGPUFallback - ``` - -#### Generating Engine Plans for x86_64 - - ```bash - /usr/src/tensorrt/bin/trtexec --onnx= --saveEngine= --int8 - ``` - -## Package Reference - -### `isaac_ros_bi3d` - -#### Bi3D Overview - -Bi3D predicts if an obstacle is within a given proximity field via a series of binary classifications; the binary classification per pixel determines if the pixel is in front of or behind the proximity field. As such, Bi3D is differentiated from other stereo disparity functions which output continuous [disparity](https://en.wikipedia.org/wiki/Binocular_disparity). Bi3D allows you to increase the diversity of functions used for obstacle detection and improve hardware diversity because Isaac ROS Proximity Segmentation is optimized to run on NVIDIA DLA hardware, which is separate from the GPU. In the form presented here, Bi3D is intended to provide proximity detections, and is not a replacement for the continuous depth estimation provided by Isaac ROS DNN Stereo Disparity. - -> **Note**: This DNN is optimized for and evaluated with RGB global shutter camera images, and accuracy may vary on monochrome images. - -#### Usage - - ```bash - ros2 launch isaac_ros_bi3d isaac_ros_bi3d.launch.py featnet_engine_file_path:= \ - segnet_engine_file_path:= - ``` - -#### Interpreting the Output - -The `isaas_ros_bi3d` package outputs a disparity image given a list of disparity values (planes). Each pixel of the output image that is not freespace is set to the value of the closest disparity plane (largest disparity value) that the pixel is deemed to be in front of. Each pixel that is predicted to be freespace is set to 0 (the furthest disparity/smallest disparity value). Freespace is defined as the region from the bottom of the image, up to the first pixel above which is not the ground plane. To find the boundary between freespace and not-freespace, one may start from the bottom of the image and, per column, find the first pixel that is not the ground plane. In the below example, the freespace of the image is shown in black: - -
Freespace Example (Original)
-
Freespace Example (Segmented)
- -The prediction of freespace eliminates the need for ground plane removal in the output image as a post-processing step, which is often applied to other stereo disparity functions. The output of `isaas_ros_bi3d` can be used to check if any pixels within the image breach a given proximity field by checking the values of all pixels. If a pixel value (disparity value) is larger than the disparity plane defining the proximity field, then it has breached that field. If a pixel does not breach any of the provided disparty planes, it is assigned a value of 0. - -#### ROS Parameters - -| ROS Parameter | Type | Default | Description | -| -------------------------- | ---------------------- | -------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `featnet_engine_file_path` | `std::string` | `'path_to_featnet_engine'` | The path to the Bi3D Featnet engine plan | -| `segnet_engine_file_path` | `std::string` | `'path_to_segnet_engine'` | The path to the Bi3D Segnet engine plan | -| `max_disparity_values` | `int64_t` | `64` | The maximum number of disparity values used for Bi3D inference. Isaac ROS Proximity Segmentation supports up to a theoretical maximum of 64 disparity values during inference. However, the maximum length of disparities that a user may run in practice is dependent on the user's hardware and availability of memory. | -| `disparity_values` | `std::vector` | `{10, 20, 30, 40, 50, 60}` | The specific threshold disparity values used for Bi3D inference. The number of disparity values must not exceed the value set in the `max_disparity_values` ROS parameter. | - -#### ROS Topics Subscribed - -| ROS Topic | Interface | Description | -| ------------------ | ---------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `left_image_bi3d` | [sensor_msgs/Image](https://github.com/ros2/common_interfaces/blob/humble/sensor_msgs/msg/Image.msg) | 1. The message must be a ROS `sensor_msgs/Image` of size **W=960, H=576** with `rgb8` image encoding.

2. There should only be a single publisher node publishing to `left_image_bi3d`. Timing behaviour with multiple publishers is not guaranteed by Bi3DNode and inference may not be performed on correct image pairs. Bi3D will process input pairs on a first available basis. Use a separate instance of Bi3DNode for each unique scene (publisher) that you wish to process. | -| `right_image_bi3d` | [sensor_msgs/Image](https://github.com/ros2/common_interfaces/blob/humble/sensor_msgs/msg/Image.msg) | 1. The message must be a ROS `sensor_msgs/Image` of size **W=960, H=576** with `rgb8` image encoding.

2. There should only be a single publisher node publishing to `right_image_bi3d`. Timing behaviour with multiple publishers is not guaranteed by Bi3DNode and inference may not be performed on correct image pairs. Bi3D will process inputs pairs on a first available basis. Use a separate instance of Bi3DNode for each unique scene (publisher) that you wish to process. | | -> Note: The images on input topics (`left_image_bi3d` and `right_image_bi3d`) should be a color image in `rgb8` format. - -#### ROS Topics Published - -| ROS Topic | Interface | Description | -| --------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `bi3d_node/bi3d_output` | [stereo_msgs/DisparityImage](https://github.com/ros2/common_interfaces/blob/humble/stereo_msgs/msg/DisparityImage.msg) | The proximity segmentation of Bi3D given as a disparity image. For pixels not deemed freespace, their value is set to the closest (largest) disparity plane that is breached. A pixel value is set to 0 if it doesn't breach any disparity plane or if it is freespace.

Output Resolution: 960x576 (WxH) | -| `bi3d_node/bi3d_disparity_values` | [isaac_ros_bi3d_interfaces/Bi3DInferenceParametersArray](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_common/blob/main/isaac_ros_bi3d_interfaces/msg/Bi3DInferenceParametersArray.msg) | The disparity values used for Bi3D inference. The timestamp is matched to the timestamp in the correpsonding output image from `bi3d_node/bi3d_output` | +--- -## Troubleshooting +### Documentation -### Isaac ROS Troubleshooting +Please visit the [Isaac ROS Documentation](https://nvidia-isaac-ros.github.io/repositories_and_packages/isaac_ros_depth_segmentation/index.html) to learn how to use this repository. -Check [here](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_common/blob/main/docs/troubleshooting.md) for solutions to problems with Isaac ROS. +--- -### DNN and Triton Troubleshooting +### Packages -Check [here](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_dnn_inference/blob/main/docs/troubleshooting.md) for solutions to problems with using DNN models and Triton. +* [`isaac_ros_bi3d`](https://nvidia-isaac-ros.github.io/repositories_and_packages/isaac_ros_depth_segmentation/isaac_ros_bi3d/index.html) + * [Quickstart](https://nvidia-isaac-ros.github.io/repositories_and_packages/isaac_ros_depth_segmentation/isaac_ros_bi3d/index.html#quickstart) + * [Try More Examples](https://nvidia-isaac-ros.github.io/repositories_and_packages/isaac_ros_depth_segmentation/isaac_ros_bi3d/index.html#try-more-examples) + * [Model Preparation](https://nvidia-isaac-ros.github.io/repositories_and_packages/isaac_ros_depth_segmentation/isaac_ros_bi3d/index.html#model-preparation) + * [Troubleshooting](https://nvidia-isaac-ros.github.io/repositories_and_packages/isaac_ros_depth_segmentation/isaac_ros_bi3d/index.html#troubleshooting) + * [API](https://nvidia-isaac-ros.github.io/repositories_and_packages/isaac_ros_depth_segmentation/isaac_ros_bi3d/index.html#api) -## Updates +### Latest -| Date | Changes | -| ---------- | ------------------------------------------------------------------ | -| 2023-05-25 | Performance improvements | -| 2023-04-05 | Source available GXF extensions | -| 2022-08-31 | Update to use latest model and to be compatible with JetPack 5.0.2 | -| 2022-06-30 | Initial release | +Update 2023-10-18: Renamed repository to `isaac_ros_depth_segmentation`. diff --git a/docs/bi3d-example.md b/docs/bi3d-example.md deleted file mode 100644 index 8e6081a..0000000 --- a/docs/bi3d-example.md +++ /dev/null @@ -1,31 +0,0 @@ -# Example Usage of Isaac ROS Proximity Segmentation - -This is an example use case of Isaac ROS Proximity Segmentation, which includes monitoring the forward zones of an autonomous mobile robot (AMR) using a HAWK stereo camera. Consider three zones in front of the robot (Zones 1-3), corresponding to different alerts and actions the robot must take if an object breaches the zone. Zone 3 generates a warning, Zone 2 sends a signal for the robot to slow, and Zone 1 results in the robot stopping immediately. - -
- -Depending on the velocity of the robot, the distances of the zones may change. For this example, we use the following distances from the front of robot to define the zones: - -| Robot Velocity | Zone 1 (Stop - Red) | Zone 2 (Slow - Yellow) | Zone 3 (Warning - Blue) | -| -------------- | ------------------- | ---------------------- | ----------------------- | -| 1 m/s | 0.5 m | 0.7 m | 4.0 m | -| 2 m/s | 2.0 m | 4.0 m | 6.0 m | -| 3 m/s | 3.5 m | 6.0 m | 9.0 m | - -The distances for each zone is converted to disparity values using the following formula: - -
- -This example uses a HAWK stereo camera with a baseline of 15 cm and focal length of 933 px. Thus, the following disparity values are calculated for each zone: - -| Robot Velocity | Zone 1 (Stop - Red) | Zone 2 (Slow - Yellow) | Zone 3 (Warning - Blue) | -| -------------- | ------------------- | ---------------------- | ----------------------- | -| 1 m/s | 280 px | 200 px | 46 px | -| 2 m/s | 70 px | 35 px | 23 px | -| 3 m/s | 40 px | 23 px | 15 px | - -This example uses the Isaac ROS Bi3D package to detect when these zones are breached. An example is shown in the table below. Pixels in red indicate that Zone 1 has been breached, yellow indicates Zone 2 has been breached, and blue indicates Zone 3 has been breached. - -| Input Scene | 1 m/s Zones | 2 m/s Zones | 3 m/s Zones | -| ------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------- | -|
|
|
|
| diff --git a/docs/tutorial-bi3d-isaac-sim.md b/docs/tutorial-bi3d-isaac-sim.md deleted file mode 100644 index 90ddb65..0000000 --- a/docs/tutorial-bi3d-isaac-sim.md +++ /dev/null @@ -1,52 +0,0 @@ -# Tutorial for Bi3D with Isaac Sim - -

- -## Overview - -This tutorial walks through setting up a graph to [segment stereo image pairs](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_proximity_segmentation) generated by Isaac Sim by depth disparity levels. For more details on the output please refer to the [interpreting the output](../README.md#interpreting-the-output) section of the [main readme](../README.md). - -Last validated with [Isaac Sim 2022.2.1](https://docs.omniverse.nvidia.com/app_isaacsim/app_isaacsim/release_notes.html#id1) - -## Tutorial Walkthrough - -1. Complete steps 1-7 listed in the [Quickstart section](../README.md#quickstart) of the main README. -2. Install and launch Isaac Sim following the steps in the [Isaac ROS Isaac Sim Setup Guide](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_common/blob/main/docs/isaac-sim-sil-setup.md) -3. Open the Isaac ROS Common USD scene (using the *Content* tab) located at: - - ```text - http://omniverse-content-production.s3-us-west-2.amazonaws.com/Assets/Isaac/2022.2.1/Isaac/Samples/ROS2/Scenario/carter_warehouse_apriltags_worker.usd - ``` - - And wait for it to load completely. -4. Go to the *Stage* tab and select `/World/Carter_ROS/ROS_Cameras/ros2_create_camera_right_info`, then in *Property* tab *-> OmniGraph Node -> Inputs -> stereoOffset X* change `0` to `-175.92`. -
-5. Enable the right camera for a stereo image pair. Go to the *Stage* tab and select `/World/Carter_ROS/ROS_Cameras/enable_camera_right`, then tick the *Condition* checkbox. -
-6. Press **Play** to start publishing data from the Isaac Sim application. -
-7. Open a second terminal and attach to the container: - - ```bash - cd ~/workspaces/isaac_ros-dev/src/isaac_ros_common && \ - ./scripts/run_dev.sh - ``` - -8. In the second terminal, start the `isaac_ros_bi3d` node using the launch files: - - ```bash - ros2 launch isaac_ros_bi3d isaac_ros_bi3d_isaac_sim.launch.py \ - featnet_engine_file_path:=/tmp/models/bi3d/bi3dnet_featnet.plan \ - segnet_engine_file_path:=/tmp/models/bi3d/bi3dnet_segnet.plan \ - max_disparity_values:=32 - ``` - - You should see a RViz window, as shown at the top of this page. - -9. Optionally, you can run the visualizer script to visualize the disparity image. - - ```bash - ros2 run isaac_ros_bi3d isaac_ros_bi3d_visualizer.py - ``` - -
diff --git a/isaac_ros_bi3d/CMakeLists.txt b/isaac_ros_bi3d/CMakeLists.txt index ee9bdfa..6523813 100644 --- a/isaac_ros_bi3d/CMakeLists.txt +++ b/isaac_ros_bi3d/CMakeLists.txt @@ -15,7 +15,7 @@ # # SPDX-License-Identifier: Apache-2.0 -cmake_minimum_required(VERSION 3.23.2) +cmake_minimum_required(VERSION 3.22.1) project(isaac_ros_bi3d LANGUAGES C CXX) if(CMAKE_COMPILER_IS_GNUCXX OR CMAKE_CXX_COMPILER_ID MATCHES "Clang") @@ -33,13 +33,9 @@ set(node_plugins "${node_plugins}nvidia::isaac_ros::bi3d::Bi3DNode;$ Value: Orbit (rviz) - Yaw: 3.9754106998443604 + Yaw: 3.2772347927093506 Saved: ~ Window Geometry: Displays: collapsed: false - Height: 968 + Height: 1066 Hide Left Dock: false Hide Right Dock: false - QMainWindow State: 000000ff00000000fd0000000400000000000001560000032efc0200000008fb0000001200530065006c0065006300740069006f006e00000001e10000009b0000005c00fffffffb0000001e0054006f006f006c002000500072006f007000650072007400690065007302000001ed000001df00000185000000a3fb000000120056006900650077007300200054006f006f02000001df000002110000018500000122fb000000200054006f006f006c002000500072006f0070006500720074006900650073003203000002880000011d000002210000017afb000000100044006900730070006c006100790073010000003b0000032e000000c700fffffffb0000002000730065006c0065006300740069006f006e00200062007500660066006500720200000138000000aa0000023a00000294fb00000014005700690064006500530074006500720065006f02000000e6000000d2000003ee0000030bfb0000000c004b0069006e0065006300740200000186000001060000030c00000261000000010000010f0000032efc0200000003fb0000001e0054006f006f006c002000500072006f00700065007200740069006500730100000041000000780000000000000000fb0000000a00560069006500770073010000003b0000032e000000a000fffffffb0000001200530065006c0065006300740069006f006e010000025a000000b200000000000000000000000200000490000000a9fc0100000001fb0000000a00560069006500770073030000004e00000080000002e100000197000000030000057c0000003efc0100000002fb0000000800540069006d006501000000000000057c0000024400fffffffb0000000800540069006d006501000000000000045000000000000000000000030b0000032e00000004000000040000000800000008fc0000000100000002000000010000000a0054006f006f006c00730100000000ffffffff0000000000000000 + Left Image: + collapsed: false + QMainWindow State: 000000ff00000000fd0000000400000000000001e000000390fc020000000afb0000001200530065006c0065006300740069006f006e00000001e10000009b0000005c00fffffffb0000001e0054006f006f006c002000500072006f007000650072007400690065007302000001ed000001df00000185000000a3fb000000120056006900650077007300200054006f006f02000001df000002110000018500000122fb000000200054006f006f006c002000500072006f0070006500720074006900650073003203000002880000011d000002210000017afb000000100044006900730070006c006100790073010000003b000001ca000000c700fffffffb0000002000730065006c0065006300740069006f006e00200062007500660066006500720200000138000000aa0000023a00000294fb00000014005700690064006500530074006500720065006f02000000e6000000d2000003ee0000030bfb0000000c004b0069006e0065006300740200000186000001060000030c00000261fb00000014004c00650066007400200049006d006100670065010000020b000000db0000002800fffffffb000000160052006900670068007400200049006d00610067006501000002ec000000df0000002800ffffff000000010000010f00000390fc0200000003fb0000001e0054006f006f006c002000500072006f00700065007200740069006500730100000041000000780000000000000000fb0000000a00560069006500770073010000003b00000390000000a000fffffffb0000001200530065006c0065006300740069006f006e010000025a000000b200000000000000000000000200000490000000a9fc0100000001fb0000000a00560069006500770073030000004e00000080000002e10000019700000003000007800000003efc0100000002fb0000000800540069006d00650100000000000007800000024400fffffffb0000000800540069006d00650100000000000004500000000000000000000004850000039000000004000000040000000800000008fc0000000100000002000000010000000a0054006f006f006c00730100000000ffffffff0000000000000000 + Right Image: + collapsed: false Selection: collapsed: false Time: @@ -164,6 +277,6 @@ Window Geometry: collapsed: false Views: collapsed: false - Width: 1404 - X: 1061 - Y: 232 + Width: 1920 + X: 0 + Y: 267 diff --git a/isaac_ros_bi3d/config/isaac_ros_bi3d_zed.rviz b/isaac_ros_bi3d/config/isaac_ros_bi3d_zed.rviz new file mode 100644 index 0000000..2c2d42d --- /dev/null +++ b/isaac_ros_bi3d/config/isaac_ros_bi3d_zed.rviz @@ -0,0 +1,197 @@ +Panels: + - Class: rviz_common/Displays + Help Height: 78 + Name: Displays + Property Tree Widget: + Expanded: + - /Global Options1 + - /Status1 + Splitter Ratio: 0.5 + Tree Height: 466 + - Class: rviz_common/Selection + Name: Selection + - Class: rviz_common/Tool Properties + Expanded: + - /2D Goal Pose1 + - /Publish Point1 + Name: Tool Properties + Splitter Ratio: 0.5886790156364441 + - Class: rviz_common/Views + Expanded: + - /Current View1 + Name: Views + Splitter Ratio: 0.5 + - Class: rviz_common/Time + Experimental: false + Name: Time + SyncMode: 0 + SyncSource: PointCloud2 +Visualization Manager: + Class: "" + Displays: + - Alpha: 0.5 + Cell Size: 1 + Class: rviz_default_plugins/Grid + Color: 160; 160; 164 + Enabled: true + Line Style: + Line Width: 0.029999999329447746 + Value: Lines + Name: Grid + Normal Cell Count: 0 + Offset: + X: 0 + Y: 0 + Z: 0 + Plane: XY + Plane Cell Count: 10 + Reference Frame: + Value: true + - Alpha: 1 + Autocompute Intensity Bounds: true + Autocompute Value Bounds: + Max Value: 10 + Min Value: -10 + Value: true + Axis: Z + Channel Name: intensity + Class: rviz_default_plugins/PointCloud2 + Color: 255; 255; 255 + Color Transformer: RGB8 + Decay Time: 0 + Enabled: true + Invert Rainbow: false + Max Color: 255; 255; 255 + Max Intensity: 4096 + Min Color: 0; 0; 0 + Min Intensity: 0 + Name: PointCloud2 + Position Transformer: XYZ + Selectable: true + Size (Pixels): 3 + Size (m): 0.009999999776482582 + Style: Flat Squares + Topic: + Depth: 5 + Durability Policy: Volatile + Filter size: 10 + History Policy: Keep Last + Reliability Policy: Reliable + Value: /points2 + Use Fixed Frame: true + Use rainbow: true + Value: true + - Class: rviz_default_plugins/Image + Enabled: true + Max Value: 1 + Median window: 5 + Min Value: 0 + Name: Image + Normalize Range: true + Topic: + Depth: 5 + Durability Policy: Volatile + History Policy: Keep Last + Reliability Policy: Reliable + Value: /zed_node/left/image_rect_color_rgb + Value: true + - Class: rviz_default_plugins/Image + Enabled: true + Max Value: 1 + Median window: 5 + Min Value: 0 + Name: Image + Normalize Range: true + Topic: + Depth: 5 + Durability Policy: Volatile + History Policy: Keep Last + Reliability Policy: Reliable + Value: /zed_node/right/image_rect_color_rgb + Value: true + Enabled: true + Global Options: + Background Color: 48; 48; 48 + Fixed Frame: zed2i_left_camera_optical_frame + Frame Rate: 30 + Name: root + Tools: + - Class: rviz_default_plugins/Interact + Hide Inactive Objects: true + - Class: rviz_default_plugins/MoveCamera + - Class: rviz_default_plugins/Select + - Class: rviz_default_plugins/FocusCamera + - Class: rviz_default_plugins/Measure + Line color: 128; 128; 0 + - Class: rviz_default_plugins/SetInitialPose + Covariance x: 0.25 + Covariance y: 0.25 + Covariance yaw: 0.06853891909122467 + Topic: + Depth: 5 + Durability Policy: Volatile + History Policy: Keep Last + Reliability Policy: Reliable + Value: /initialpose + - Class: rviz_default_plugins/SetGoal + Topic: + Depth: 5 + Durability Policy: Volatile + History Policy: Keep Last + Reliability Policy: Reliable + Value: /goal_pose + - Class: rviz_default_plugins/PublishPoint + Single click: true + Topic: + Depth: 5 + Durability Policy: Volatile + History Policy: Keep Last + Reliability Policy: Reliable + Value: /clicked_point + Transformation: + Current: + Class: rviz_default_plugins/TF + Value: true + Views: + Current: + Class: rviz_default_plugins/Orbit + Distance: 14.81318187713623 + Enable Stereo Rendering: + Stereo Eye Separation: 0.05999999865889549 + Stereo Focal Distance: 1 + Swap Stereo Eyes: false + Value: false + Focal Point: + X: -0.536247968673706 + Y: 0.23787975311279297 + Z: 1.2995383739471436 + Focal Shape Fixed Size: true + Focal Shape Size: 0.05000000074505806 + Invert Z Axis: false + Name: Current View + Near Clip Distance: 0.009999999776482582 + Pitch: -0.7852040529251099 + Target Frame: + Value: Orbit (rviz) + Yaw: 4.648578643798828 + Saved: ~ +Window Geometry: + Displays: + collapsed: false + Height: 1136 + Hide Left Dock: false + Hide Right Dock: false + Image: + collapsed: false + QMainWindow State: 000000ff00000000fd0000000400000000000003b1000003d6fc020000000afb0000001200530065006c0065006300740069006f006e00000001e10000009b0000005c00fffffffb0000001e0054006f006f006c002000500072006f007000650072007400690065007302000001ed000001df00000185000000a3fb000000120056006900650077007300200054006f006f02000001df000002110000018500000122fb000000200054006f006f006c002000500072006f0070006500720074006900650073003203000002880000011d000002210000017afb000000100044006900730070006c006100790073000000003b0000025b000000c700fffffffb0000002000730065006c0065006300740069006f006e00200062007500660066006500720200000138000000aa0000023a00000294fb00000014005700690064006500530074006500720065006f02000000e6000000d2000003ee0000030bfb0000000c004b0069006e0065006300740200000186000001060000030c00000261fb0000000a0049006d006100670065010000003b000001e70000002800fffffffb0000000a0049006d0061006700650100000228000001e90000002800ffffff000000010000010f000003d6fc0200000003fb0000001e0054006f006f006c002000500072006f00700065007200740069006500730100000041000000780000000000000000fb0000000a00560069006500770073000000003b000003d6000000a000fffffffb0000001200530065006c0065006300740069006f006e010000025a000000b200000000000000000000000200000490000000a9fc0100000001fb0000000a00560069006500770073030000004e00000080000002e10000019700000003000007380000003efc0100000002fb0000000800540069006d00650100000000000007380000024400fffffffb0000000800540069006d0065010000000000000450000000000000000000000381000003d600000004000000040000000800000008fc0000000100000002000000010000000a0054006f006f006c00730100000000ffffffff0000000000000000 + Selection: + collapsed: false + Time: + collapsed: false + Tool Properties: + collapsed: false + Views: + collapsed: false + Width: 1848 + X: 72 + Y: 27 diff --git a/isaac_ros_bi3d/config/zed.yaml b/isaac_ros_bi3d/config/zed.yaml new file mode 100644 index 0000000..d06a05f --- /dev/null +++ b/isaac_ros_bi3d/config/zed.yaml @@ -0,0 +1,163 @@ +# config/common_yaml +# Common parameters to Stereolabs ZED and ZED mini cameras +# +# Note: the parameter svo_file is passed as exe argumet +--- +/**: + ros__parameters: + + general: + svo_file: "" # usually overwritten by launch file + svo_loop: false # Enable loop mode when using an SVO as input source + svo_realtime: true # if true the SVO will be played trying to respect the original framerate eventually skipping frames, otherwise every frame will be processed respecting the `pub_frame_rate` setting + camera_timeout_sec: 5 + camera_max_reconnect: 5 + camera_flip: false + zed_id: 0 # IMPORTANT: to be used in simulation to distinguish individual cameras im multi-camera configurations - usually overwritten by launch file + serial_number: 0 # usually overwritten by launch file + pub_resolution: 'MEDIUM' # The resolution used for output. 'HD2K', 'HD1080', 'HD1200', 'HD720', 'MEDIUM', 'SVGA', 'VGA', 'LOW' + pub_frame_rate: 15.0 # [DYNAMIC] - frequency of publishing of visual images and depth images + gpu_id: -1 + region_of_interest: '[]' # A polygon defining the ROI where the ZED SDK perform the processing ignoring the rest. Coordinates must be normalized to '1.0' to be resolution independent. + #region_of_interest: '[[0.25,0.33],[0.75,0.33],[0.75,0.5],[0.5,0.75],[0.25,0.5]]' # A polygon defining the ROI where the ZED SDK perform the processing ignoring the rest. Coordinates must be normalized to '1.0' to be resolution independent. + #region_of_interest: '[[0.25,0.25],[0.75,0.25],[0.75,0.75],[0.25,0.75]]' # A polygon defining the ROI where the ZED SDK perform the processing ignoring the rest. Coordinates must be normalized to '1.0' to be resolution independent. + #region_of_interest: '[[0.5,0.25],[0.75,0.5],[0.5,0.75],[0.25,0.5]]' # A polygon defining the ROI where the ZED SDK perform the processing ignoring the rest. Coordinates must be normalized to '1.0' to be resolution independent. + sdk_verbose: 1 + + video: + brightness: 4 # [DYNAMIC] Not available for ZED X/ZED X Mini + contrast: 4 # [DYNAMIC] Not available for ZED X/ZED X Mini + hue: 0 # [DYNAMIC] Not available for ZED X/ZED X Mini + saturation: 4 # [DYNAMIC] + sharpness: 4 # [DYNAMIC] + gamma: 8 # [DYNAMIC] + auto_exposure_gain: true # [DYNAMIC] + exposure: 80 # [DYNAMIC] + gain: 80 # [DYNAMIC] + auto_whitebalance: true # [DYNAMIC] + whitebalance_temperature: 42 # [DYNAMIC] - [28,65] works only if `auto_whitebalance` is false + qos_history: 1 # '1': KEEP_LAST - '2': KEEP_ALL + qos_depth: 1 # Queue size if using KEEP_LAST + qos_reliability: 1 # '1': RELIABLE - '2': BEST_EFFORT - + qos_durability: 2 # '1': TRANSIENT_LOCAL - '2': VOLATILE + + depth: + depth_mode: 'NONE' # Matches the ZED SDK setting: 'NONE', 'PERFORMANCE', 'QUALITY', 'ULTRA', 'NEURAL' - Note: if 'NONE' all the modules that requires depth extraction are disabled by default (Pos. Tracking, Obj. Detection, Mapping, ...) + depth_stabilization: 1 # Forces positional tracking to start if major than 0 - Range: [0,100] + openni_depth_mode: false # 'false': 32bit float [meters], 'true': 16bit unsigned int [millimeters] + point_cloud_freq: 10.0 # [DYNAMIC] - frequency of the pointcloud publishing (equal or less to `grab_frame_rate` value) + depth_confidence: 50 # [DYNAMIC] + depth_texture_conf: 100 # [DYNAMIC] + remove_saturated_areas: true # [DYNAMIC] + qos_history: 1 # '1': KEEP_LAST - '2': KEEP_ALL + qos_depth: 1 # Queue size if using KEEP_LAST + qos_reliability: 1 # '1': RELIABLE - '2': BEST_EFFORT - + qos_durability: 2 # '1': TRANSIENT_LOCAL - '2': VOLATILE + + pos_tracking: + pos_tracking_enabled: false # True to enable positional tracking from start + imu_fusion: true # enable/disable IMU fusion. When set to false, only the optical odometry will be used. + publish_tf: true # [usually overwritten by launch file] publish `odom -> base_link` TF + publish_map_tf: true # [usually overwritten by launch file] publish `map -> odom` TF + publish_imu_tf: true # [usually overwritten by launch file] enable/disable the IMU TF broadcasting + base_frame: "base_link" # usually overwritten by launch file + map_frame: "map" + odometry_frame: "odom" + area_memory_db_path: "" + area_memory: true # Enable to detect loop closure + depth_min_range: 0.0 # Set this value for removing fixed zones of the robot in the FoV of the camerafrom the visual odometry evaluation + set_as_static: false # If 'true' the camera will be static and not move in the environment + set_gravity_as_origin: true # If 'true' align the positional tracking world to imu gravity measurement. Keep the yaw from the user initial pose. + floor_alignment: false # Enable to automatically calculate camera/floor offset + initial_base_pose: [0.0, 0.0, 0.0, 0.0, 0.0, 0.0] # Initial position of the `base_frame` in the map -> [X, Y, Z, R, P, Y] + init_odom_with_first_valid_pose: true # Enable to initialize the odometry with the first valid pose + path_pub_rate: 2.0 # [DYNAMIC] - Camera trajectory publishing frequency + path_max_count: -1 # use '-1' for unlimited path size + two_d_mode: false # Force navigation on a plane. If true the Z value will be fixed to "fixed_z_value", roll and pitch to zero + fixed_z_value: 0.00 # Value to be used for Z coordinate if `two_d_mode` is true + transform_time_offset: 0.0 # The value added to the timestamp of `map->odom` and `odom->base_link`` transform being generated + qos_history: 1 # '1': KEEP_LAST - '2': KEEP_ALL + qos_depth: 1 # Queue size if using KEEP_LAST + qos_reliability: 1 # '1': RELIABLE - '2': BEST_EFFORT + qos_durability: 2 # '1': TRANSIENT_LOCAL - '2': VOLATILE + + gnss_fusion: + gnss_fusion_enabled: false # fuse 'sensor_msg/NavSatFix' message information into pose data + gnss_fix_topic: "/gps/fix" # Name of the GNSS topic of type NavSatFix to subscribe [Default: "/gps/fix"] + gnss_init_distance: 5.0 # The minimum distance traveled by the robot required to initilize the GNSS fusion + gnss_zero_altitude: false # Set to `true` to ignore GNSS altitude information + gnss_frame: "gnss_link" # [usually overwritten by launch file] The TF frame of the GNSS sensor + publish_utm_tf: true # Publish `utm` -> `map` TF + broadcast_utm_transform_as_parent_frame: false # if 'true' publish `utm` -> `map` TF, otherwise `map` -> `utm` + + mapping: + mapping_enabled: false # True to enable mapping and fused point cloud pubblication + resolution: 0.05 # maps resolution in meters [min: 0.01f - max: 0.2f] + max_mapping_range: 5.0 # maximum depth range while mapping in meters (-1 for automatic calculation) [2.0, 20.0] + fused_pointcloud_freq: 1.0 # frequency of the publishing of the fused colored point cloud + clicked_point_topic: "/clicked_point" # Topic published by Rviz when a point of the cloud is clicked. Used for plane detection + qos_history: 1 # '1': KEEP_LAST - '2': KEEP_ALL + qos_depth: 1 # Queue size if using KEEP_LAST + qos_reliability: 1 # '1': RELIABLE - '2': BEST_EFFORT - + qos_durability: 2 # '1': TRANSIENT_LOCAL - '2': VOLATILE + + sensors: + sensors_image_sync: false # Synchronize Sensors messages with latest published video/depth message + sensors_pub_rate: 200. # frequency of publishing of sensors data. MAX: 400. - MIN: grab rate + qos_history: 1 # '1': KEEP_LAST - '2': KEEP_ALL + qos_depth: 1 # Queue size if using KEEP_LAST + qos_reliability: 1 # '1': RELIABLE - '2': BEST_EFFORT - + qos_durability: 2 # '1': TRANSIENT_LOCAL - '2': VOLATILE + + object_detection: + od_enabled: false # True to enable Object Detection + model: 'MULTI_CLASS_BOX_MEDIUM' # 'MULTI_CLASS_BOX_FAST', 'MULTI_CLASS_BOX_MEDIUM', 'MULTI_CLASS_BOX_ACCURATE', 'PERSON_HEAD_BOX_FAST', 'PERSON_HEAD_BOX_ACCURATE' + allow_reduced_precision_inference: true # Allow inference to run at a lower precision to improve runtime and memory usage + max_range: 20.0 # [m] Defines a upper depth range for detections + confidence_threshold: 50.0 # [DYNAMIC] - Minimum value of the detection confidence of an object [0,99] + prediction_timeout: 0.5 # During this time [sec], the object will have OK state even if it is not detected. Set this parameter to 0 to disable SDK predictions + filtering_mode: 1 # '0': NONE - '1': NMS3D - '2': NMS3D_PER_CLASS + mc_people: false # [DYNAMIC] - Enable/disable the detection of persons for 'MULTI_CLASS_X' models + mc_vehicle: true # [DYNAMIC] - Enable/disable the detection of vehicles for 'MULTI_CLASS_X' models + mc_bag: true # [DYNAMIC] - Enable/disable the detection of bags for 'MULTI_CLASS_X' models + mc_animal: true # [DYNAMIC] - Enable/disable the detection of animals for 'MULTI_CLASS_X' models + mc_electronics: true # [DYNAMIC] - Enable/disable the detection of electronic devices for 'MULTI_CLASS_X' models + mc_fruit_vegetable: true # [DYNAMIC] - Enable/disable the detection of fruits and vegetables for 'MULTI_CLASS_X' models + mc_sport: true # [DYNAMIC] - Enable/disable the detection of sport-related objects for 'MULTI_CLASS_X' models + qos_history: 1 # '1': KEEP_LAST - '2': KEEP_ALL + qos_depth: 1 # Queue size if using KEEP_LAST + qos_reliability: 1 # '1': RELIABLE - '2': BEST_EFFORT + qos_durability: 2 # '1': TRANSIENT_LOCAL - '2': VOLATILE + + body_tracking: + bt_enabled: false # True to enable Body Tracking + model: 'HUMAN_BODY_MEDIUM' # 'HUMAN_BODY_FAST', 'HUMAN_BODY_MEDIUM', 'HUMAN_BODY_ACCURATE' + body_format: 'BODY_38' # 'BODY_18','BODY_34','BODY_38','BODY_70' + allow_reduced_precision_inference: false # Allow inference to run at a lower precision to improve runtime and memory usage + max_range: 20.0 # [m] Defines a upper depth range for detections + body_kp_selection: 'FULL' # 'FULL', 'UPPER_BODY' + enable_body_fitting: false # Defines if the body fitting will be applied + enable_tracking: true # Defines if the object detection will track objects across images flow + prediction_timeout_s: 0.5 # During this time [sec], the skeleton will have OK state even if it is not detected. Set this parameter to 0 to disable SDK predictions + confidence_threshold: 50.0 # [DYNAMIC] - Minimum value of the detection confidence of skeleton key points [0,99] + minimum_keypoints_threshold: 5 # [DYNAMIC] - Minimum number of skeleton key points to be detected for a valid skeleton + qos_history: 1 # '1': KEEP_LAST - '2': KEEP_ALL + qos_depth: 1 # Queue size if using KEEP_LAST + qos_reliability: 1 # '1': RELIABLE - '2': BEST_EFFORT + qos_durability: 2 # '1': TRANSIENT_LOCAL - '2': VOLATILE + + use_sim_time: false # EXPERIMENTAL (only for development) - Set to true to enable SIMULATION mode # + sim_address: '192.168.1.90' # EXPERIMENTAL (only for development) - The local address of the machine running the simulator + + debug: + debug_common: false + debug_video_depth: false + debug_camera_controls: false + debug_point_cloud: false + debug_positional_tracking: false + debug_gnss: false + debug_sensors: false + debug_mapping : false + debug_terrain_mapping : false + debug_object_detection : false + debug_body_tracking : false \ No newline at end of file diff --git a/isaac_ros_bi3d/gxf/CMakeLists.txt b/isaac_ros_bi3d/gxf/CMakeLists.txt new file mode 100644 index 0000000..9434857 --- /dev/null +++ b/isaac_ros_bi3d/gxf/CMakeLists.txt @@ -0,0 +1,94 @@ +# SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES +# Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +project(gxf_cvcore_bi3d LANGUAGES C CXX) + +if(CMAKE_COMPILER_IS_GNUCXX OR CMAKE_CXX_COMPILER_ID MATCHES "Clang") + add_compile_options(-fPIC -w) +endif() + +enable_language(CUDA) + +# Dependencies +find_package(CUDAToolkit) +find_package(yaml-cpp) +find_package(GXF ${ISAAC_ROS_GXF_VERSION} MODULE REQUIRED + COMPONENTS + core + cuda + multimedia + serialization +) +find_package(TENSORRT) +find_package(Eigen3) +find_package(isaac_ros_image_proc REQUIRED) + +set(CMAKE_INCLUDE_CURRENT_DIR TRUE) + + +# Create extension +add_library(gxf_cvcore_bi3d SHARED + extensions/bi3d/bi3d.cpp + extensions/bi3d/components/bi3d_postprocessor.cpp + extensions/bi3d/components/bi3d_inference.cpp + extensions/bi3d/components/bi3d_postprocessor.cu + extensions/bi3d/components/bi3d_depthview.cpp + + extensions/bi3d/inference/Bi3D_detail.cpp + extensions/bi3d/inference/Bi3D_detail.hpp + extensions/bi3d/inference/Bi3D.cpp + extensions/bi3d/inference/Bi3D.h + extensions/bi3d/inference/Bi3DPreProcessor.cpp + extensions/bi3d/inference/Bi3DPostProcessor.cpp +) +target_include_directories(gxf_cvcore_bi3d PRIVATE ${isaac_ros_image_proc_INCLUDE_DIRS}) + +add_library(corelib STATIC + # Inferencer (Bi3D only) + gems/dnn_inferencer/inferencer/TensorRTInferencer.cpp + gems/dnn_inferencer/inferencer/TensorRTUtils.cpp + gems/dnn_inferencer/inferencer/Inferencer.cpp + gems/dnn_inferencer/inferencer/Errors.cpp +) +target_include_directories(corelib PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} ${isaac_ros_image_proc_INCLUDE_DIRS}) + +target_link_libraries(corelib PUBLIC + Eigen3::Eigen + GXF::core + GXF::cuda + GXF::isaac_messages + GXF::multimedia + CUDA::cudart + CUDA::nppc + CUDA::nppial + CUDA::nppicc + CUDA::nppidei + CUDA::nppif + CUDA::nppig + CUDA::nppisu + CUDA::nppitc + TENSORRT::nvinfer + yaml-cpp + +) + +target_compile_options(gxf_cvcore_bi3d PUBLIC -fPIC) + +target_link_libraries(gxf_cvcore_bi3d + corelib + isaac_ros_image_proc::gxf_tensorops +) \ No newline at end of file diff --git a/isaac_ros_bi3d/gxf/bi3d/3dv/src/Bi3D_detail.h b/isaac_ros_bi3d/gxf/bi3d/3dv/src/Bi3D_detail.h deleted file mode 100644 index df612c3..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/3dv/src/Bi3D_detail.h +++ /dev/null @@ -1,340 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -#ifndef CVCORE_BI3D_DETAIL_H_ -#define CVCORE_BI3D_DETAIL_H_ - -#include - -#include -#include -#include - -#include -#include - -namespace cvcore { namespace bi3d { namespace detail { - -using DisparityLevels = Array; -using InputImage = Tensor; -using PreprocessedImage = traits::to_planar_t>; -using BatchPreprocessedImage = traits::add_batch_t; -using FeatureResponse = Tensor; -using BatchFeatureResponse = traits::add_batch_t; -using ConfidenceMap = Tensor; -using GuidedConfidence = traits::to_cx_t; -using RefinedConfidence = Tensor; -using DisparityConfidence = traits::to_cx_t; -using QuantizedDisparity = traits::to_u8_t; - -using NormalizeFunctor = tensor_ops::ImageToNormalizedPlanarTensorOperator; - -void _pad(Tensor & dst, const Tensor & src, - int topBorderHeight, int leftBorderWidth, cudaStream_t stream); - -template -void Pad(Tensor & dst, Tensor & src, - const Array & topBorderHeights, const Array & leftBorderWidths, - cudaStream_t stream) -{ - using TensorType = Tensor; - - for(std::size_t c = 0; c < dst.getChannelCount(); ++c) - { - TensorType _dst{dst.getWidth(), dst.getHeight(), - dst.getData() + c*dst.getStride(TensorDimension::CHANNEL), - dst.isCPU()}; - TensorType _src{src.getWidth(), src.getHeight(), - src.getData() + c*src.getStride(TensorDimension::CHANNEL), - src.isCPU()}; - - _pad(_dst, _src, topBorderHeights[c], leftBorderWidths[c], stream); - } -} - -template -void Pad(Tensor & dst, Tensor & src, - const Array & topBorderHeights, int leftBorderWidth, cudaStream_t stream) -{ - using TensorType = Tensor; - - for(std::size_t c = 0; c < dst.getChannelCount(); ++c) - { - TensorType _dst{dst.getWidth(), dst.getHeight(), - dst.getData() + c*dst.getStride(TensorDimension::CHANNEL), - dst.isCPU()}; - TensorType _src{src.getWidth(), src.getHeight(), - src.getData() + c*src.getStride(TensorDimension::CHANNEL), - src.isCPU()}; - - _pad(_dst, _src, topBorderHeights[c], leftBorderWidth, stream); - } -} - -template -void Pad(Tensor & dst, Tensor & src, - int topBorderHeight, const Array & leftBorderWidths, cudaStream_t stream) -{ - using TensorType = Tensor; - - for(std::size_t c = 0; c < dst.getChannelCount(); ++c) - { - TensorType _dst{dst.getWidth(), dst.getHeight(), - dst.getData() + c*dst.getStride(TensorDimension::CHANNEL), - dst.isCPU()}; - TensorType _src{src.getWidth(), src.getHeight(), - src.getData() + c*src.getStride(TensorDimension::CHANNEL), - src.isCPU()}; - - _pad(_dst, _src, topBorderHeight, leftBorderWidths[c], stream); - } -} - -template -void Pad(Tensor & dst, Tensor & src, - int topBorderHeight, int leftBorderWidth, cudaStream_t stream) -{ - using TensorType = Tensor; - - for(std::size_t c = 0; c < dst.getChannelCount(); ++c) - { - TensorType _dst{dst.getWidth(), dst.getHeight(), - dst.getData() + c*dst.getStride(TensorDimension::CHANNEL), - dst.isCPU()}; - TensorType _src{src.getWidth(), src.getHeight(), - src.getData() + c*src.getStride(TensorDimension::CHANNEL), - src.isCPU()}; - - _pad(_dst, _src, topBorderHeight, leftBorderWidth, stream); - } -} - -template -void Pad(Tensor & dst, Tensor & src, - int topBorderHeight, int leftBorderWidth, cudaStream_t stream) -{ - using TensorType = Tensor; - - for(std::size_t n = 0; n < dst.getDepth(); ++n) - { - TensorType _dst{dst.getWidth(), dst.getHeight(), dst.getChannelCount(), - dst.getData() + n*dst.getStride(TensorDimension::DEPTH), - dst.isCPU()}; - TensorType _src{src.getWidth(), src.getHeight(), src.getChannelCount(), - src.getData() + n*src.getStride(TensorDimension::DEPTH), - src.isCPU()}; - - Pad(_dst, _src, topBorderHeight, leftBorderWidth, stream); - } -} - -void _threshold(Tensor & dst, const Tensor & src, - float valueLow, float valueHigh, float thresholdValue, - cudaStream_t stream); - -template -void Threshold(Tensor & dst, Tensor & src, - float valueLow, float valueHigh, float thresholdValue, - cudaStream_t stream) -{ - using TensorType = Tensor; - - TensorType _dst{dst.getWidth(), dst.getHeight() * dst.getChannelCount(), - dst.getData(), dst.isCPU()}; - TensorType _src{src.getWidth(), src.getHeight() * src.getChannelCount(), - src.getData(), src.isCPU()}; - - _threshold(_dst, _src, valueLow, valueHigh, thresholdValue, stream); -} - -template -void Threshold(Tensor & dst, Tensor & src, - float valueLow, float valueHigh, float thresholdValue, - cudaStream_t stream) -{ - using TensorType = Tensor; - - TensorType _dst{dst.getWidth(), dst.getHeight(), - dst.getChannelCount() * dst.getDepth(), - dst.getData(), dst.isCPU()}; - TensorType _src{src.getWidth(), src.getHeight(), - src.getChannelCount() * src.getDepth(), - src.getData(), src.isCPU()}; - - Threshold(_dst, _src, valueLow, valueHigh, thresholdValue, stream); -} - -template -void CropAndResize(Tensor & dst, Tensor & src, - const BBox & dstROI, const BBox & srcROI, - tensor_ops::InterpolationType type, cudaStream_t stream) -{ - using TensorType = Tensor; - - for(std::size_t c = 0; c < dst.getChannelCount(); ++c) - { - TensorType _dst{dst.getWidth(), dst.getHeight(), - dst.getData() + c*dst.getStride(TensorDimension::CHANNEL), - dst.isCPU()}; - TensorType _src{src.getWidth(), src.getHeight(), - src.getData() + c*src.getStride(TensorDimension::CHANNEL), - src.isCPU()}; - - tensor_ops::CropAndResize(_dst, _src, dstROI, srcROI, type, stream); - } -} - -template -void CropAndResize(Tensor & dst, Tensor & src, - const BBox & dstROI, const BBox & srcROI, - tensor_ops::InterpolationType type, cudaStream_t stream) -{ - using TensorType = Tensor; - - for(std::size_t n = 0; n < dst.getDepth(); ++n) - { - TensorType _dst{dst.getWidth(), dst.getHeight(), dst.getChannelCount(), - dst.getData() + n*dst.getStride(TensorDimension::DEPTH), - dst.isCPU()}; - TensorType _src{src.getWidth(), src.getHeight(), src.getChannelCount(), - src.getData() + n*src.getStride(TensorDimension::DEPTH), - src.isCPU()}; - - CropAndResize(_dst, _src, dstROI, srcROI, type, stream); - } -} - -template -void Resize(Tensor & dst, Tensor & src, - tensor_ops::InterpolationType type, cudaStream_t stream) -{ - using TensorType = Tensor; - - for(std::size_t c = 0; c < dst.getChannelCount(); ++c) - { - TensorType _dst{dst.getWidth(), dst.getHeight(), - dst.getData() + c*dst.getStride(TensorDimension::CHANNEL), - dst.isCPU()}; - TensorType _src{src.getWidth(), src.getHeight(), - src.getData() + c*src.getStride(TensorDimension::CHANNEL), - src.isCPU()}; - - tensor_ops::Resize(_dst, _src, false, type, stream); - } -} - -template -void Resize(Tensor & dst, Tensor & src, - tensor_ops::InterpolationType type, cudaStream_t stream) -{ - using TensorType = Tensor; - - for(std::size_t n = 0; n < dst.getDepth(); ++n) - { - TensorType _dst{dst.getWidth(), dst.getHeight(), dst.getChannelCount(), - dst.getData() + n*dst.getStride(TensorDimension::DEPTH), - dst.isCPU()}; - TensorType _src{src.getWidth(), src.getHeight(), src.getChannelCount(), - src.getData() + n*src.getStride(TensorDimension::DEPTH), - src.isCPU()}; - - Resize(_dst, _src, type, stream); - } -} - -void _clear(Tensor & dst, cudaStream_t stream); - -template -void Clear(Tensor & dst, cudaStream_t stream) -{ - using TensorType = Tensor; - - for(std::size_t c = 0; c < dst.getChannelCount(); ++c) - { - TensorType _dst{dst.getWidth(), dst.getHeight(), - dst.getData() + c*dst.getStride(TensorDimension::CHANNEL), - dst.isCPU()}; - - _clear(_dst, stream); - } -} - -template -void ConvertBitDepth(Tensor & dst, Tensor & src, - float scale, cudaStream_t stream) -{ - using TensorSrcType = Tensor; - using TensorDstType = traits::to_u8_t; - - for(std::size_t c = 0; c < dst.getChannelCount(); ++c) - { - TensorDstType _dst{dst.getWidth(), dst.getHeight(), - dst.getData() + c*dst.getStride(TensorDimension::CHANNEL), - dst.isCPU()}; - TensorSrcType _src{src.getWidth(), src.getHeight(), - src.getData() + c*src.getStride(TensorDimension::CHANNEL), - src.isCPU()}; - - tensor_ops::ConvertBitDepth(_dst, _src, scale, stream); - } -} - -void _sigmoid(Tensor & dst, Tensor & src, - cudaStream_t stream); - -void _sigmoid(Tensor & dst, Tensor & src); - -template -void Sigmoid(Tensor & dst, Tensor & src, - cudaStream_t stream) -{ - using TensorType = Tensor; - - for(std::size_t c = 0; c < dst.getChannelCount(); ++c) - { - TensorType _dst{dst.getWidth(), dst.getHeight(), - dst.getData() + c*dst.getStride(TensorDimension::CHANNEL), - dst.isCPU()}; - TensorType _src{src.getWidth(), src.getHeight(), - src.getData() + c*src.getStride(TensorDimension::CHANNEL), - src.isCPU()}; - - _sigmoid(_dst, _src, stream); - } -} - -template -void Sigmoid(Tensor & dst, Tensor & src) -{ - using TensorType = Tensor; - - for(std::size_t c = 0; c < dst.getChannelCount(); ++c) - { - TensorType _dst{dst.getWidth(), dst.getHeight(), - dst.getData() + c*dst.getStride(TensorDimension::CHANNEL), - dst.isCPU()}; - TensorType _src{src.getWidth(), src.getHeight(), - src.getData() + c*src.getStride(TensorDimension::CHANNEL), - src.isCPU()}; - - _sigmoid(_dst, _src); - } -} - -}}} // namespace cvcore::bi3d::detail - -#endif // CVCORE_BI3D_DETAIL_H_ diff --git a/isaac_ros_bi3d/gxf/bi3d/CMakeLists.txt b/isaac_ros_bi3d/gxf/bi3d/CMakeLists.txt deleted file mode 100644 index 52c607a..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/CMakeLists.txt +++ /dev/null @@ -1,135 +0,0 @@ -# SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -# Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# SPDX-License-Identifier: Apache-2.0 - -project(gxf_cvcore_bi3d LANGUAGES C CXX CUDA) - -if(CMAKE_COMPILER_IS_GNUCXX OR CMAKE_CXX_COMPILER_ID MATCHES "Clang") - add_compile_options(-fPIC -w) -endif() - -# Dependencies -find_package(CUDAToolkit) -include(YamlCpp) -find_package(GXF ${ISAAC_ROS_GXF_VERSION} MODULE REQUIRED - COMPONENTS - core - cuda - multimedia - serialization - std -) -find_package(TENSORRT) - -# Create extension -add_library(gxf_cvcore_bi3d SHARED - extensions/bi3d/Bi3D.cpp - extensions/bi3d/Bi3D.hpp - extensions/bi3d/Bi3DRegistry.cpp - - extensions/tensor_ops/ImageAdapter.cpp - extensions/tensor_ops/ImageAdapter.hpp - extensions/tensor_ops/ImageUtils.cpp - extensions/tensor_ops/ImageUtils.hpp - - extensions/tensor_ops/detail/ImageAdapterTensorImpl.cpp - extensions/tensor_ops/detail/ImageAdapterTensorImpl.hpp - extensions/tensor_ops/detail/ImageAdapterVideoBufferImpl.cpp - extensions/tensor_ops/detail/ImageAdapterVideoBufferImpl.hpp -) - -set(CMAKE_INCLUDE_CURRENT_DIR TRUE) -target_include_directories(gxf_cvcore_bi3d PRIVATE - ${CMAKE_CURRENT_SOURCE_DIR}/cvcore/include - ${CMAKE_CURRENT_SOURCE_DIR}/3dv/include) - - -add_library(cvcore_bi3d STATIC - # Tensorops - cvcore/src/tensor_ops/ArithmeticOperations.cpp - cvcore/src/tensor_ops/BBoxUtils.cpp - cvcore/src/tensor_ops/ColorConversions.cpp - cvcore/src/tensor_ops/DBScan.cpp - cvcore/src/tensor_ops/Errors.cpp - cvcore/src/tensor_ops/Filters.cpp - cvcore/src/tensor_ops/Filters.h - cvcore/src/tensor_ops/FusedOperations.cpp - cvcore/src/tensor_ops/GeometryTransforms.cpp - cvcore/src/tensor_ops/IImageWarp.cpp - cvcore/src/tensor_ops/NppUtils.cpp - cvcore/src/tensor_ops/NppUtils.h - cvcore/src/tensor_ops/OneEuroFilter.cpp - cvcore/src/tensor_ops/TensorOperators.cpp - - # Core - cvcore/src/core/cvcore/Array.cpp - cvcore/src/core/cvcore/Dummy.cu - cvcore/src/core/cvcore/MathTypes.cpp - cvcore/src/core/cvcore/Tensor.cpp - cvcore/src/core/utility/CVError.cpp - cvcore/src/core/utility/Instrumentation.cpp - cvcore/src/core/utility/Memory.cpp - cvcore/src/core/utility/ProfileUtils.cpp - - # Inferencer (Bi3d only) - cvcore/src/inferencer/tensorrt/TensorRTInferencer.cpp - cvcore/src/inferencer/Inferencer.cpp - cvcore/src/inferencer/Errors.cpp - cvcore/src/inferencer/tensorrt/TensorRTUtils.h - cvcore/src/inferencer/tensorrt/TensorRTUtils.cpp - cvcore/src/inferencer/tensorrt/TensorRTInferencer.h - - # TRTBackend - cvcore/src/trtbackend/TRTBackend.cpp -) - -target_include_directories(cvcore_bi3d PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/cvcore/include) -target_compile_options(cvcore_bi3d PUBLIC -fPIC) -target_link_libraries(cvcore_bi3d PUBLIC - CUDA::cudart - CUDA::nppc - CUDA::nppial - CUDA::nppicc - CUDA::nppidei - CUDA::nppif - CUDA::nppig - CUDA::nppisu - CUDA::nppitc - TENSORRT::nvinfer -) - -add_library(bi3d_3dv STATIC - 3dv/src/Bi3D_detail.cpp - 3dv/src/Bi3D_detail.h - 3dv/src/Bi3D.cpp - 3dv/src/Bi3DPostProcessor.cpp - 3dv/src/Bi3DPreProcessor.cpp -) -target_include_directories(bi3d_3dv PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/3dv/include) -target_compile_options(bi3d_3dv PUBLIC -fPIC) -target_link_libraries(bi3d_3dv PUBLIC - cvcore_bi3d -) -target_link_libraries(gxf_cvcore_bi3d - PUBLIC - GXF::cuda - GXF::multimedia - GXF::std - yaml-cpp - PRIVATE - cvcore_bi3d - bi3d_3dv -) diff --git a/isaac_ros_bi3d/gxf/bi3d/bi3d_postprocessor/CMakeLists.txt b/isaac_ros_bi3d/gxf/bi3d/bi3d_postprocessor/CMakeLists.txt deleted file mode 100644 index e165480..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/bi3d_postprocessor/CMakeLists.txt +++ /dev/null @@ -1,51 +0,0 @@ -# SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -# Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# SPDX-License-Identifier: Apache-2.0 - -cmake_minimum_required(VERSION 3.23.2) -project(gxf_bi3d_postprocessor LANGUAGES C CXX) - -# Create extension -add_library(gxf_bi3d_postprocessor SHARED - bi3d_postprocessor_extension.cpp - bi3d_postprocess/bi3d_message_splitter.cpp - bi3d_postprocess/bi3d_message_splitter.hpp - bi3d_postprocessor_utils.hpp - bi3d_postprocess/bi3d_postprocessor.cpp - bi3d_postprocess/bi3d_postprocessor.cu.cpp - bi3d_postprocess/bi3d_postprocessor.cu.hpp - bi3d_postprocess/bi3d_postprocessor.hpp -) - -# Mark as CUDA files with non-standard extensions -set_source_files_properties( - bi3d_postprocess/bi3d_postprocessor.cu.cpp - bi3d_postprocess/bi3d_postprocessor.cu.hpp - PROPERTIES - LANGUAGE CUDA -) - -find_package(CUDAToolkit REQUIRED) - -set(CMAKE_INCLUDE_CURRENT_DIR TRUE) - -target_link_libraries(gxf_bi3d_postprocessor - PUBLIC - GXF::cuda - GXF::multimedia - GXF::std - yaml-cpp -) diff --git a/isaac_ros_bi3d/gxf/bi3d/bi3d_postprocessor/bi3d_postprocess/bi3d_message_splitter.cpp b/isaac_ros_bi3d/gxf/bi3d/bi3d_postprocessor/bi3d_postprocess/bi3d_message_splitter.cpp deleted file mode 100644 index 00ad8f7..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/bi3d_postprocessor/bi3d_postprocess/bi3d_message_splitter.cpp +++ /dev/null @@ -1,67 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -#include "gxf/std/parameter_parser_std.hpp" -#include "gxf/std/timestamp.hpp" -#include "gxf/multimedia/video.hpp" -#include "bi3d_message_splitter.hpp" - -namespace nvidia -{ -namespace isaac_ros -{ - -gxf_result_t Bi3DMessageSplitter::registerInterface(gxf::Registrar * registrar) noexcept -{ - gxf::Expected result; - result &= registrar->parameter( - receiver_, "receiver", "Receiver", - "Message from Bi3D post processor"); - result &= registrar->parameter( - disparity_image_transmitter_, "disparity_image_transmitter", "Disparity image transmitter", - "The collapsed output of Bi3D"); - result &= registrar->parameter( - disparity_values_transmitter_, "disparity_values_transmitter", "Disparity values transmitter", - "The disparity values used for Bi3D inference"); - return gxf::ToResultCode(result); -} - -gxf_result_t Bi3DMessageSplitter::tick() noexcept -{ - // Receive bi3d post processor output - auto bi3d_postprocessed_message = receiver_->receive(); - if (!bi3d_postprocessed_message) { - GXF_LOG_ERROR("Failed to get message"); - return gxf::ToResultCode(bi3d_postprocessed_message); - } - - // Publish message - auto result = disparity_image_transmitter_->publish(bi3d_postprocessed_message.value()); - if (!result) { - return gxf::ToResultCode(result); - } - result = disparity_values_transmitter_->publish(bi3d_postprocessed_message.value()); - if (!result) { - return gxf::ToResultCode(result); - } - - return GXF_SUCCESS; - -} - -} // namespace isaac_ros -} // namespace nvidia diff --git a/isaac_ros_bi3d/gxf/bi3d/bi3d_postprocessor/bi3d_postprocess/bi3d_message_splitter.hpp b/isaac_ros_bi3d/gxf/bi3d/bi3d_postprocessor/bi3d_postprocess/bi3d_message_splitter.hpp deleted file mode 100644 index b642f30..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/bi3d_postprocessor/bi3d_postprocess/bi3d_message_splitter.hpp +++ /dev/null @@ -1,50 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -#ifndef NVIDIA_ISAAC_ROS_EXTENSIONS_BI3D_MESSAGE_SPLITTER_HPP_ -#define NVIDIA_ISAAC_ROS_EXTENSIONS_BI3D_MESSAGE_SPLITTER_HPP_ - -#include "gxf/std/allocator.hpp" -#include "gxf/std/codelet.hpp" -#include "gxf/std/receiver.hpp" -#include "gxf/std/transmitter.hpp" - - -namespace nvidia -{ -namespace isaac_ros -{ - -// GXF codelet that takes the postprocessed output of Bi3D and splits it into disparity image and disparity values -class Bi3DMessageSplitter : public gxf::Codelet -{ -public: - gxf_result_t registerInterface(gxf::Registrar * registrar) noexcept override; - gxf_result_t start() noexcept override {return GXF_SUCCESS;} - gxf_result_t tick() noexcept override; - gxf_result_t stop() noexcept override {return GXF_SUCCESS;} - -private: - gxf::Parameter> receiver_; - gxf::Parameter> disparity_image_transmitter_; - gxf::Parameter> disparity_values_transmitter_; -}; - -} // namespace isaac_ros -} // namespace nvidia - -#endif // NVIDIA_ISAAC_ROS_EXTENSIONS_BI3D_MESSAGE_SPLITTER_HPP_ diff --git a/isaac_ros_bi3d/gxf/bi3d/bi3d_postprocessor/bi3d_postprocess/bi3d_postprocessor.cpp b/isaac_ros_bi3d/gxf/bi3d/bi3d_postprocessor/bi3d_postprocess/bi3d_postprocessor.cpp deleted file mode 100644 index 740eeba..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/bi3d_postprocessor/bi3d_postprocess/bi3d_postprocessor.cpp +++ /dev/null @@ -1,178 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -#include "gxf/std/parameter_parser_std.hpp" -#include "gxf/std/timestamp.hpp" -#include "bi3d_postprocessor.hpp" -#include "bi3d_postprocessor_utils.hpp" -#include "bi3d_postprocessor.cu.hpp" - - -namespace nvidia -{ -namespace isaac_ros -{ - -gxf_result_t Bi3DPostprocessor::registerInterface(gxf::Registrar * registrar) noexcept -{ - gxf::Expected result; - result &= registrar->parameter( - bi3d_receiver_, "bi3d_receiver", "Bi3D input", - "Tensor from Bi3D"); - result &= registrar->parameter( - output_transmitter_, "output_transmitter", "Output transmitter", - "The collapsed output of Bi3D"); - result &= registrar->parameter( - pool_, "pool", "Pool", - "Allocator instance for output videobuffer"); - result &= registrar->parameter( - disparity_values_, "disparity_values", "Disparity values", - "Fixed disparity values used for Bi3D inference", - gxf::Registrar::NoDefaultParameter(), GXF_PARAMETER_FLAGS_OPTIONAL); - result &= registrar->parameter( - disparity_tensor_name_, "disparity_tensor_name", "Disparity tensor name", - "Name of the disparity tensor from Bi3D"); - result &= registrar->parameter( - disparity_values_tensor_name_, "disparity_values_tensor_name", "Disparity values tensor name", - "Name of the (dynamic) disparity values tensor from Bi3D", - gxf::Registrar::NoDefaultParameter(), GXF_PARAMETER_FLAGS_OPTIONAL); - return gxf::ToResultCode(result); -} - -gxf_result_t Bi3DPostprocessor::start() noexcept -{ - auto fixed_disparity_values = disparity_values_.try_get(); - auto disparity_values_name = disparity_values_tensor_name_.try_get(); - if (fixed_disparity_values && disparity_values_name) { - GXF_LOG_WARNING( - "Both fixed disparity values and dynamic disparity values given. Fixed disparity values will be used."); - } - return GXF_SUCCESS; -} - -gxf_result_t Bi3DPostprocessor::tick() noexcept -{ - // Receive bi3d output - auto bi3d_message = bi3d_receiver_->receive(); - if (!bi3d_message) { - GXF_LOG_ERROR("Failed to get message"); - return gxf::ToResultCode(bi3d_message); - } - - // Create output message - auto out_message = gxf::Entity::New(context()); - if (!out_message) { - GXF_LOG_ERROR("Failed to allocate message"); - return gxf::ToResultCode(out_message); - } - - // Get Bi3D disparity tensor - auto bi3d_disparity_tensor = bi3d_message->get(disparity_tensor_name_.get().c_str()); - if (!bi3d_disparity_tensor) { - GXF_LOG_ERROR("Failed to get disparity image tensor"); - return gxf::ToResultCode(bi3d_disparity_tensor); - } - if (!bi3d_disparity_tensor.value()->data()) { - GXF_LOG_ERROR("Failed to get pointer to bi3d disparity data"); - return gxf::ToResultCode(bi3d_disparity_tensor.value()->data()); - } - const float * bi3d_disparity_data_ptr = bi3d_disparity_tensor.value()->data().value(); - - - // Get disparity values used for Bi3D inference - int disparity_count; - const int * disparity_values_data; - - auto fixed_disparity_values = disparity_values_.try_get(); - if (fixed_disparity_values) { - disparity_count = fixed_disparity_values.value().size(); - disparity_values_data = fixed_disparity_values.value().data(); - } else { - auto disparity_values_name = disparity_values_tensor_name_.try_get(); - if (!disparity_values_name) { - GXF_LOG_ERROR("Neither dynamic nor fixed disparity values specified"); - return GXF_FAILURE; - } - auto bi3d_disparity_value_tensor = bi3d_message->get( - disparity_values_name.value().c_str()); - if (!bi3d_disparity_value_tensor) { - GXF_LOG_ERROR("Failed to get disparity values tensor"); - return gxf::ToResultCode(bi3d_disparity_value_tensor); - } - disparity_count = bi3d_disparity_value_tensor.value()->element_count(); - disparity_values_data = bi3d_disparity_value_tensor.value()->data().value(); - - // Add dynamic disparity value tensor to output message - auto forwarded_disp_values = out_message.value().add( - bi3d_disparity_value_tensor->name()); - *forwarded_disp_values.value() = std::move(*bi3d_disparity_value_tensor.value()); - } - - // Get dimensions of tensor - gxf::Shape dims = bi3d_disparity_tensor.value()->shape(); - const int image_height = dims.dimension(1); - const int image_width = dims.dimension(2); - const int image_size = dims.dimension(1) * dims.dimension(2); - - // Create video buffer - auto out_video_buffer = out_message.value().add(); - if (!out_video_buffer) { - GXF_LOG_ERROR("Failed to allocate output video buffer"); - return gxf::ToResultCode(out_video_buffer); - } - - // Allocate video buffer on device (unpadded gray32) - auto maybe_allocation = AllocateVideoBuffer( - out_video_buffer.value(), image_width, image_height, gxf::MemoryStorageType::kDevice, - pool_.get()); - if (!maybe_allocation) { - GXF_LOG_ERROR("Failed to allocate output video buffer's memory."); - return gxf::ToResultCode(maybe_allocation); - } - - cudaMemset( - out_video_buffer.value()->pointer(), 0, - image_size * bi3d_disparity_tensor.value()->bytes_per_element()); - - for (int i = 0; i < disparity_count; i++) { - cuda_postprocess( - bi3d_disparity_data_ptr + (i * image_size), - (float *)out_video_buffer.value()->pointer(), disparity_values_data[i], image_height, - image_width); - } - - // Add timestamp - std::string timestamp_name{"timestamp"}; - auto maybe_bi3d_timestamp = bi3d_message->get(); - if (!maybe_bi3d_timestamp) { - GXF_LOG_ERROR("Failed to get a timestamp from Bi3D output"); - } - auto out_timestamp = out_message.value().add(timestamp_name.c_str()); - if (!out_timestamp) {return GXF_FAILURE;} - *out_timestamp.value() = *maybe_bi3d_timestamp.value(); - - // Publish message - auto result = output_transmitter_->publish(out_message.value()); - if (!result) { - return gxf::ToResultCode(result); - } - - return GXF_SUCCESS; -} - -} // namespace isaac_ros -} // namespace nvidia diff --git a/isaac_ros_bi3d/gxf/bi3d/bi3d_postprocessor/bi3d_postprocess/bi3d_postprocessor.cu.cpp b/isaac_ros_bi3d/gxf/bi3d/bi3d_postprocessor/bi3d_postprocess/bi3d_postprocessor.cu.cpp deleted file mode 100644 index 931525b..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/bi3d_postprocessor/bi3d_postprocess/bi3d_postprocessor.cu.cpp +++ /dev/null @@ -1,54 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 -#include "bi3d_postprocessor.cu.hpp" - -namespace nvidia -{ -namespace isaac_ros -{ - -__global__ void postprocessing_kernel( - const float * input, float * output, int disparity, - int imageHeight, int imageWidth) -{ - const uint32_t x = blockIdx.x * blockDim.x + threadIdx.x; - const uint32_t y = blockIdx.y * blockDim.y + threadIdx.y; - const uint32_t index = y * imageWidth + x; - if (x < imageWidth && y < imageHeight) { - if (input[index] == 1.0) { - output[index] = input[index] * disparity; - } - } -} - -uint16_t ceil_div(uint16_t numerator, uint16_t denominator) -{ - uint32_t accumulator = numerator + denominator - 1; - return accumulator / denominator; -} - -void cuda_postprocess( - const float * input, float * output, int disparity, int imageHeight, - int imageWidth) -{ - dim3 block(16, 16); - dim3 grid(ceil_div(imageWidth, 16), ceil_div(imageHeight, 16), 1); - postprocessing_kernel << < grid, block >> > (input, output, disparity, imageHeight, imageWidth); -} - -} // namespace isaac_ros -} // namespace nvidia diff --git a/isaac_ros_bi3d/gxf/bi3d/bi3d_postprocessor/bi3d_postprocess/bi3d_postprocessor.hpp b/isaac_ros_bi3d/gxf/bi3d/bi3d_postprocessor/bi3d_postprocess/bi3d_postprocessor.hpp deleted file mode 100644 index 6fb5e28..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/bi3d_postprocessor/bi3d_postprocess/bi3d_postprocessor.hpp +++ /dev/null @@ -1,55 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -#ifndef NVIDIA_ISAAC_ROS_EXTENSIONS_BI3D_POSTPROCESSOR_HPP_ -#define NVIDIA_ISAAC_ROS_EXTENSIONS_BI3D_POSTPROCESSOR_HPP_ - -#include - -#include "gxf/std/allocator.hpp" -#include "gxf/std/codelet.hpp" -#include "gxf/std/receiver.hpp" -#include "gxf/std/transmitter.hpp" - - -namespace nvidia -{ -namespace isaac_ros -{ - -// GXF codelet that takes Bi3D output tensor list and collapses it into single segmentation image -class Bi3DPostprocessor : public gxf::Codelet -{ -public: - gxf_result_t registerInterface(gxf::Registrar * registrar) noexcept override; - gxf_result_t start() noexcept override; - gxf_result_t tick() noexcept override; - gxf_result_t stop() noexcept override {return GXF_SUCCESS;} - -private: - gxf::Parameter> bi3d_receiver_; - gxf::Parameter> output_transmitter_; - gxf::Parameter> pool_; - gxf::Parameter> disparity_values_; - gxf::Parameter disparity_tensor_name_; - gxf::Parameter disparity_values_tensor_name_; -}; - -} // namespace isaac_ros -} // namespace nvidia - -#endif // NVIDIA_ISAAC_ROS_EXTENSIONS_BI3D_POSTPROCESSOR_HPP_ diff --git a/isaac_ros_bi3d/gxf/bi3d/bi3d_postprocessor/bi3d_postprocessor_extension.cpp b/isaac_ros_bi3d/gxf/bi3d/bi3d_postprocessor/bi3d_postprocessor_extension.cpp deleted file mode 100644 index ebe5ee6..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/bi3d_postprocessor/bi3d_postprocessor_extension.cpp +++ /dev/null @@ -1,45 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 -#include - -#include "gxf/core/gxf.h" -#include "gxf/std/extension_factory_helper.hpp" -#include "bi3d_postprocess/bi3d_postprocessor.hpp" -#include "bi3d_postprocess/bi3d_message_splitter.hpp" - -extern "C" { - -GXF_EXT_FACTORY_BEGIN() - -GXF_EXT_FACTORY_SET_INFO( - 0xb764a9fce34d11ec, 0x8fea0242ac120002, "Bi3DPostprocessorExtension", - "Bi3D Post Processing GXF extension", - "NVIDIA", "1.0.0", "LICENSE"); - -GXF_EXT_FACTORY_ADD( - 0xa8a24ecde2544d6a, 0x9a4b81cdb71085d6, - nvidia::isaac_ros::Bi3DPostprocessor, nvidia::gxf::Codelet, - "Collapses Bi3D output tensorlist to single segmentation image"); - -GXF_EXT_FACTORY_ADD( - 0xa8a24ecde2564d6a, 0x9a4b89cdb71085d6, - nvidia::isaac_ros::Bi3DMessageSplitter, nvidia::gxf::Codelet, - "Splits Bi3D postprocessor output into disparity image and disparity values"); - -GXF_EXT_FACTORY_END() - -} // extern "C" diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/core/Array.h b/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/core/Array.h deleted file mode 100644 index c9f23d8..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/core/Array.h +++ /dev/null @@ -1,386 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2019-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -#ifndef CVCORE_ARRAY_H -#define CVCORE_ARRAY_H - -#include -#include -#include - -namespace cvcore { - -/** - * Base implementation of Array. - */ -class ArrayBase -{ -public: - /** - * Constructor of a non-owning arrays. - * @param capacity capacity of the array. - * @param elemSize byte size of each element. - * @param dataPtr data pointer to the raw source array. - * @param isCPU whether to allocate the array on CPU or GPU. - */ - ArrayBase(std::size_t capacity, std::size_t elemSize, void *dataPtr, bool isCPU); - - /** - * Constructor of a memory-owning arrays - * @param capacity capacity of the array. - * @param elemSize byte size of each element. - * @param isCPU whether to allocate the array on CPU or GPU. - */ - ArrayBase(std::size_t capacity, std::size_t elemSize, bool isCPU); - - /** - * Destructor of ArrayBase. - */ - ~ArrayBase(); - - /** - * ArrayBase is non-copyable. - */ - ArrayBase(const ArrayBase &) = delete; - - /** - * ArrayBase is non-copyable. - */ - ArrayBase &operator=(const ArrayBase &) = delete; - - /** - * Move constructor of ArrayBase. - */ - ArrayBase(ArrayBase &&); - - /** - * Move assignment operator of ArrayBase. - */ - ArrayBase &operator=(ArrayBase &&); - - /** - * Get the pointer to specified index. - * @param idx element index. - * @return pointer to the specified element. - */ - void *getElement(int idx) const; - - /** - * Get the size of the array. - * @return size of the array. - */ - std::size_t getSize() const; - - /** - * Get the capacity of the array. - * @return size of the array. - */ - std::size_t getCapacity() const; - - /** - * Get the size of each element. - * @return size of each element. - */ - std::size_t getElementSize() const; - - /** - * Set the size of the array. - * @param size size of the array. - */ - void setSize(std::size_t size); - - /** - * Get the flag whether the array is CPU or GPU array. - * @return whether the array is CPU array. - */ - bool isCPU() const; - - /** - * Get the flag whether the array is owning memory space. - * @return whether the array owns memory. - */ - bool isOwning() const; - - /** - * Get the raw pointer to the array data. - * @return void pointer to the first element of the array. - */ - void *getData() const; - -private: - ArrayBase(); - - void *m_data; - std::size_t m_size; - std::size_t m_capacity; - std::size_t m_elemSize; - bool m_isOwning; - bool m_isCPU; -}; - -/** - * Implementation of Array class. - * @tparam T type of element in array. - */ -template -class Array : public ArrayBase -{ -public: - /** - * Default constructor of an array. - */ - Array() - : ArrayBase{0, sizeof(T), nullptr, true} - { - } - - /** - * Constructor of a non-owning array. - * @param size size of the array. - * @param capacity capacity of the array. - * @param dataPtr data pointer to the raw source array. - * @param isCPU whether to allocate array on CPU or GPU. - */ - Array(std::size_t size, std::size_t capacity, void *dataPtr, bool isCPU = true) - : ArrayBase{capacity, sizeof(T), dataPtr, isCPU} - { - ArrayBase::setSize(size); - } - - /** - * Constructor of a memory-owning array. - * @param capacity capacity of the array. - * @param isCPU whether to allocate array on CPU or GPU. - */ - Array(std::size_t capacity, bool isCPU = true) - : ArrayBase{capacity, sizeof(T), isCPU} - { - } - - /** - * Destructor of the Array. - */ - ~Array() - { - // call resize here such that CPU-based destructor - // will call destructors of the objects stored - // in the array before deallocating the storage - setSize(0); - } - - /** - * Array is non-copyable. - */ - Array(const Array &) = delete; - - /** - * Array is non-copyable. - */ - Array &operator=(const Array &) = delete; - - /** - * Move constructor of Array. - */ - Array(Array &&t) - : Array() - { - *this = std::move(t); - } - - /** - * Move assignment operator of Array. - */ - Array &operator=(Array &&t) - { - static_cast(*this) = std::move(t); - return *this; - } - - /** - * Set size of the Array. - * @param size size of the Array. - */ - void setSize(std::size_t size) - { - const std::size_t oldSize = getSize(); - ArrayBase::setSize(size); - if (isCPU()) - { - // shrinking case - for (std::size_t i = size; i < oldSize; ++i) - { - reinterpret_cast(getElement(i))->~T(); - } - // expanding case - for (std::size_t i = oldSize; i < size; ++i) - { - new (getElement(i)) T; - } - } - } - - /** - * Const array index operator. - * @param idx index of element. - * @return const reference to the specified element. - */ - const T &operator[](int idx) const - { - assert(idx >= 0 && idx < getSize()); - return *reinterpret_cast(getElement(idx)); - } - - /** - * Array index operator. - * @param idx index of element. - * @return reference to the specified element. - */ - T &operator[](int idx) - { - assert(idx >= 0 && idx < getSize()); - return *reinterpret_cast(getElement(idx)); - } -}; - -/** - * Implementation of ArrayN class. - * @tparam T type of element in array. - * @tparam N capacity of array. - */ -template -class ArrayN : public ArrayBase -{ -public: - /** - * Default constructor of ArrayN (create an owning Tensor with capacity N). - */ - ArrayN() - : ArrayBase{N, sizeof(T), true} - { - setSize(N); - } - - /** - * Constructor of a non-owning ArrayN. - * @param size size of the array. - * @param dataPtr data pointer to the raw source array. - * @param isCPU whether to allocate array on CPU or GPU. - */ - ArrayN(std::size_t size, void *dataPtr, bool isCPU = true) - : ArrayBase{N, sizeof(T), dataPtr, isCPU} - { - ArrayBase::setSize(size); - } - - /** - * Constructor of a memory-owning ArrayN. - * @param isCPU whether to allocate array on CPU or GPU. - */ - ArrayN(bool isCPU) - : ArrayBase{N, sizeof(T), isCPU} - { - setSize(N); - } - - /** - * Destructor of the ArrayN. - */ - ~ArrayN() - { - // call resize here such that CPU-based destructor - // will call destructors of the objects stored - // in the array before deallocating the storage - setSize(0); - } - - /** - * ArrayN is non-copyable. - */ - ArrayN(const ArrayN &) = delete; - - /** - * ArrayN is non-copyable. - */ - ArrayN &operator=(const ArrayN &) = delete; - - /** - * Move constructor of ArrayN. - */ - ArrayN(ArrayN &&t) - : ArrayN() - { - *this = std::move(t); - } - - /** - * Move assignment operator of ArrayN. - */ - ArrayN &operator=(ArrayN &&t) - { - static_cast(*this) = std::move(t); - return *this; - } - - /** - * Set size of the ArrayN. - * @param size size of the ArrayN. - */ - void setSize(std::size_t size) - { - const std::size_t oldSize = getSize(); - ArrayBase::setSize(size); - if (isCPU()) - { - // shrinking case - for (std::size_t i = size; i < oldSize; ++i) - { - reinterpret_cast(getElement(i))->~T(); - } - // expanding case - for (std::size_t i = oldSize; i < size; ++i) - { - new (getElement(i)) T; - } - } - } - - /** - * Const ArrayN index operator. - * @param idx index of element. - * @return const reference to the specified element. - */ - const T &operator[](int idx) const - { - assert(idx >= 0 && idx < getSize()); - return *reinterpret_cast(getElement(idx)); - } - - /** - * ArrayN index operator. - * @param idx index of element. - * @return reference to the specified element. - */ - T &operator[](int idx) - { - assert(idx >= 0 && idx < getSize()); - return *reinterpret_cast(getElement(idx)); - } -}; - -} // namespace cvcore - -#endif // CVCORE_ARRAY_H diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/core/BBox.h b/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/core/BBox.h deleted file mode 100644 index 93100d3..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/core/BBox.h +++ /dev/null @@ -1,142 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2020-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -#ifndef CVCORE_BBOX_H -#define CVCORE_BBOX_H - -#include -#include -#include - -namespace cvcore { - -/** - * A struct. - * Structure used to store bounding box. - */ -struct BBox -{ - int xmin{0}; /**< minimum x coordinate. */ - int ymin{0}; /**< minimum y coordinate. */ - int xmax{0}; /**< maximum x coordinate. */ - int ymax{0}; /**< maximum y coordinate. */ - - /** - * Clamp a bounding box based on a restricting clamp box - * @param Clamping bounding box (xmin, ymin, xmax, ymax) - * @return Clamped bounding box - */ - BBox clamp(const BBox &clampBox) const - { - BBox outbox; - outbox.xmin = std::max(clampBox.xmin, xmin); - outbox.xmax = std::min(clampBox.xmax, xmax); - outbox.ymin = std::max(clampBox.ymin, ymin); - outbox.ymax = std::min(clampBox.ymax, ymax); - return outbox; - } - - /** - * @return Width of the bounding box - */ - size_t getWidth() const - { - return xmax - xmin; - } - - /** - * @return Height of the bounding box - */ - size_t getHeight() const - { - return ymax - ymin; - } - - /** - * Checks if the bounding box is valid. - */ - bool isValid() const - { - return (xmin < xmax) && (ymin < ymax) && (getWidth() > 0) && (getHeight() > 0); - } - - /** - * Returns the center of the bounding box - * @return X,Y coordinate tuple - */ - std::pair getCenter() const - { - int centerX = xmin + getWidth() / 2; - int centerY = ymin + getHeight() / 2; - return std::pair(centerX, centerY); - } - - /** - * Scales bounding box based along the width and height retaining the same center. - * @param Scale in X direction along the width - * @param Scale in Y direction along the height - * @return Scaled bounding box - */ - BBox scale(float scaleW, float scaleH) const - { - auto center = getCenter(); - float newW = getWidth() * scaleW; - float newH = getHeight() * scaleH; - BBox outbox; - outbox.xmin = center.first - newW / 2; - outbox.xmax = center.first + newW / 2; - outbox.ymin = center.second - newH / 2; - outbox.ymax = center.second + newH / 2; - - return outbox; - } - - /** - * Resizes bounding box to a square bounding box based on - * the longest edge and clamps the bounding box based on the limits provided. - * @param Clamping bounding box (xmin, ymin, xmax, ymax) - * @return Sqaure bounding box - */ - BBox squarify(const BBox &clampBox) const - { - size_t w = getWidth(); - size_t h = getHeight(); - - BBox clampedBox1 = clamp(clampBox); - if (!clampedBox1.isValid()) - { - throw std::range_error("Invalid bounding box generated\n"); - } - float scaleW = static_cast(std::max(w, h)) / w; - float scaleH = static_cast(std::max(w, h)) / h; - BBox scaledBBox = clampedBox1.scale(scaleW, scaleH); - BBox clampedBox2 = scaledBBox.clamp(clampBox); - if (!clampedBox2.isValid()) - { - throw std::range_error("Invalid bounding box generated\n"); - } - size_t newW = clampedBox2.getWidth(); - size_t newH = clampedBox2.getHeight(); - size_t minW = std::min(newH, newW); - clampedBox2.ymax = clampedBox2.ymin + minW; - clampedBox2.xmax = clampedBox2.xmin + minW; - return clampedBox2; - } -}; - -} // namespace cvcore -#endif // CVCORE_BBOX_H diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/core/CVError.h b/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/core/CVError.h deleted file mode 100644 index 82c16c1..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/core/CVError.h +++ /dev/null @@ -1,116 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2020-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -#ifndef CVCORE_CVERROR_H -#define CVCORE_CVERROR_H - -#include -#include -#include -#include -#include - -#include - -namespace cvcore { - -// CVCORE ERROR CODES -// ----------------------------------------------------------------------------- -// Defining the CVCORE Error Codes on std::error_condition -// std::error_condition creates a set of sub-system independent codes which may -// be used to describe ANY downstream error in a broad sense. An std::error_code -// is defined within the sub-system context (i.e. tensor_ops, trtbackend, ...) -// which is mapped to the cvcore::ErrorCode. -// As an example, cvcore::ErrorCode -1 may not ABSOLUTELY mean the same as -// tensor_ops::FaultCode -1, but does mean the same as tensor_ops:FaultCode 4. -// Thus, tensor_ops::FaultCode 4 needs to be mapped to cvcore::ErrorCode -1. -enum class ErrorCode : std::int32_t -{ - SUCCESS = 0, - NOT_READY, - NOT_IMPLEMENTED, - INVALID_ARGUMENT, - INVALID_IMAGE_FORMAT, - INVALID_STORAGE_TYPE, - INVALID_ENGINE_TYPE, - INVALID_OPERATION, - DETECTED_NAN_IN_RESULT, - OUT_OF_MEMORY, - DEVICE_ERROR, - SYSTEM_ERROR, -}; - -} // namespace cvcore - -// WARNING: Extending base C++ namespace to cover cvcore error codes -namespace std { - -template<> -struct is_error_condition_enum : true_type -{ -}; - -template<> -struct is_error_code_enum : true_type -{ -}; - -} // namespace std - -namespace cvcore { - -std::error_condition make_error_condition(ErrorCode) noexcept; - -std::error_code make_error_code(ErrorCode) noexcept; - -// ----------------------------------------------------------------------------- - -inline void CheckCudaError(cudaError_t code, const char *file, const int line) -{ - if (code != cudaSuccess) - { - const char *errorMessage = cudaGetErrorString(code); - const std::string message = "CUDA error returned at " + std::string(file) + ":" + std::to_string(line) + - ", Error code: " + std::to_string(code) + " (" + std::string(errorMessage) + ")"; - throw std::runtime_error(message); - } -} - -inline void CheckErrorCode(std::error_code err, const char *file, const int line) -{ - const std::string message = "Error returned at " + std::string(file) + ":" + std::to_string(line) + - ", Error code: " + std::string(err.message()); - - if (err != cvcore::make_error_code(cvcore::ErrorCode::SUCCESS)) - { - throw std::runtime_error(message); - } -} - -} // namespace cvcore - -#define CHECK_ERROR(val) \ - { \ - cvcore::CheckCudaError((val), __FILE__, __LINE__); \ - } - -#define CHECK_ERROR_CODE(val) \ - { \ - cvcore::CheckErrorCode((val), __FILE__, __LINE__); \ - } - -#endif // CVCORE_CVERROR_H diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/core/CameraModel.h b/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/core/CameraModel.h deleted file mode 100644 index 157acf4..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/core/CameraModel.h +++ /dev/null @@ -1,292 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -#ifndef CVCORE_CAMERAMODEL_H -#define CVCORE_CAMERAMODEL_H - -#include - -#include "cv/core/Array.h" -#include "cv/core/MathTypes.h" - -namespace cvcore { - -/** - * An enum. - * Enum type for Camera Distortion type. - */ -enum class CameraDistortionType : uint8_t -{ - UNKNOWN, /**< Unknown arbitrary distortion model. */ - NONE, /**< No distortion applied. */ - Polynomial, /**< Polynomial distortion model. */ - FisheyeEquidistant, /**< Equidistant Fisheye distortion model. */ - FisheyeEquisolid, /**< Equisolid Fisheye distortion model. */ - FisheyeOrthoGraphic, /**< Orthographic Fisheye distortion model. */ - FisheyeStereographic /**< Stereographic Fisheye distortion model. */ -}; - -/** - * Struct type used to store Camera Distortion model type and coefficients. - */ -struct CameraDistortionModel -{ - CameraDistortionType type; /**< Camera distortion model type. */ - union /**< Camera distortion model coefficients. */ - { - float coefficients[8]; - struct - { - float k1, k2, k3, k4, k5, k6; - float p1, p2; - }; - }; - - CameraDistortionModel() - : type(CameraDistortionType::UNKNOWN), - k1(0.0), k2(0.0), k3(0.0), k4(0.0), k5(0.0), k6(0.0), - p1(0.0), p2(0.0) {} - - /** - * Camera Distortion Model creation using array of coefficients. - * @param distortionType Camera distortion model type - * @param distortionCoefficients An array of camera distortion model coefficients - * @return Camera Distortion Model - */ - CameraDistortionModel(CameraDistortionType distortionType, std::array & distortionCoefficients) - : type(distortionType) - { - std::copy(distortionCoefficients.begin(), distortionCoefficients.end(), std::begin(coefficients)); - } - - /** - * Camera Distortion Model creation using individual coefficients. - * @param distortionType Camera distortion model type - * @param k1 Camera distortion model coefficient - k1 - * @param k2 Camera distortion model coefficient - k2 - * @param k3 Camera distortion model coefficient - k3 - * @param k4 Camera distortion model coefficient - k4 - * @param k5 Camera distortion model coefficient - k5 - * @param k6 Camera distortion model coefficient - k6 - * @param p1 Camera distortion model coefficient - p1 - * @param p2 Camera distortion model coefficient - p2 - * @return Camera Distortion Model - */ - CameraDistortionModel(CameraDistortionType distortionType, float k1, float k2, float k3, \ - float k4, float k5, float k6, float p1, float p2) - : type(distortionType) - , k1(k1) - , k2(k2) - , k3(k3) - , k4(k4) - , k5(k5) - , k6(k6) - , p1(p1) - , p2(p2) - { - - } - - /** - * Get camera distortion model type. - * @return Camera distortion model type - */ - CameraDistortionType getDistortionType() const - { - return type; - } - - /** - * Get camera distortion model coefficients. - * @return Camera distortion model coefficients array - */ - const float * getCoefficients() const - { - return &coefficients[0]; - } - - inline bool operator==(const CameraDistortionModel & other) const noexcept - { - return this->k1 == other.k1 && - this->k2 == other.k2 && - this->k3 == other.k3 && - this->k4 == other.k4 && - this->k5 == other.k5 && - this->k6 == other.k6 && - this->p1 == other.p1 && - this->p2 == other.p2; - } - - inline bool operator!=(const CameraDistortionModel & other) const noexcept - { - return !(*this == other); - } -}; - -/** - * Struct type used to store Camera Intrinsics. - */ -struct CameraIntrinsics -{ - CameraIntrinsics() = default; - - /** - * Camera Instrinsics creation with given intrinsics values - * @param fx Camera axis x focal length in pixels - * @param fy Camera axis y focal length in pixels - * @param cx Camera axis x principal point in pixels - * @param cy Camera axis y principal point in pixels - * @param s Camera slanted pixel - * @return Camera Intrinsics - */ - CameraIntrinsics(float fx, float fy, float cx, float cy, float s = 0.0) - { - m_intrinsics[0][0] = fx; - m_intrinsics[0][1] = s; - m_intrinsics[0][2] = cx; - m_intrinsics[1][0] = 0.0; - m_intrinsics[1][1] = fy; - m_intrinsics[1][2] = cy; - } - - /** - * Get camera intrinsics x focal length. - * @return Camera x focal length - */ - float fx() const - { - return m_intrinsics[0][0]; - } - - /** - * Get camera intrinsics y focal length. - * @return Camera y focal length - */ - float fy() const - { - return m_intrinsics[1][1]; - } - - /** - * Get camera intrinsics x principal point. - * @return Camera x principal point - */ - float cx() const - { - return m_intrinsics[0][2]; - } - - /** - * Get camera intrinsics y principal point. - * @return Camera y principal point - */ - float cy() const - { - return m_intrinsics[1][2]; - } - - /** - * Get camera intrinsics slanted pixels. - * @return Camera slanted pixels - */ - float skew() const - { - return m_intrinsics[0][1]; - } - - /** - * Get camera intrinsics 2D array. - * @return Camera intrisics array - */ - const float * getMatrix23() const - { - return &m_intrinsics[0][0]; - } - - inline bool operator==(const CameraIntrinsics & other) const noexcept - { - return m_intrinsics[0][0] == other.m_intrinsics[0][0] && - m_intrinsics[0][1] == other.m_intrinsics[0][1] && - m_intrinsics[0][2] == other.m_intrinsics[0][2] && - m_intrinsics[1][0] == other.m_intrinsics[1][0] && - m_intrinsics[1][1] == other.m_intrinsics[1][1] && - m_intrinsics[1][2] == other.m_intrinsics[1][2]; - } - - inline bool operator!=(const CameraIntrinsics & other) const noexcept - { - return !(*this == other); - } - - float m_intrinsics[2][3] {{1.0, 0.0, 0.0},{0.0, 1.0, 0.0}}; /**< Camera intrinsics 2D arrat. */ -}; - -/** - * Struct type used to store Camera Extrinsics. - */ -struct CameraExtrinsics -{ - using RawMatrixType = float[3][4]; - - CameraExtrinsics() = default; - - /** - * Camera Extrinsics creation with given extrinsics as raw 2D [3 x 4] array - * @param extrinsics Camera extrinsics as raw 2D array - * @return Camera Extrinsics - */ - explicit CameraExtrinsics(const RawMatrixType & extrinsics) - { - std::copy(&extrinsics[0][0], &extrinsics[0][0] + 3 * 4, &m_extrinsics[0][0]); - } - - inline bool operator==(const CameraExtrinsics & other) const noexcept - { - return m_extrinsics[0][0] == other.m_extrinsics[0][0] && - m_extrinsics[0][1] == other.m_extrinsics[0][1] && - m_extrinsics[0][2] == other.m_extrinsics[0][2] && - m_extrinsics[0][3] == other.m_extrinsics[0][3] && - m_extrinsics[1][0] == other.m_extrinsics[1][0] && - m_extrinsics[1][1] == other.m_extrinsics[1][1] && - m_extrinsics[1][2] == other.m_extrinsics[1][2] && - m_extrinsics[1][3] == other.m_extrinsics[1][3] && - m_extrinsics[2][0] == other.m_extrinsics[2][0] && - m_extrinsics[2][1] == other.m_extrinsics[2][1] && - m_extrinsics[2][2] == other.m_extrinsics[2][2] && - m_extrinsics[2][3] == other.m_extrinsics[2][3]; - } - - inline bool operator!=(const CameraExtrinsics & other) const noexcept - { - return !(*this == other); - } - - RawMatrixType m_extrinsics {{1.0, 0.0, 0.0, 0.0}, - {0.0, 1.0, 0.0, 0.0}, - {0.0, 0.0, 1.0, 0.0}}; -}; - -struct CameraModel -{ - CameraIntrinsics intrinsic; - CameraExtrinsics extrinsic; - CameraDistortionModel distortion; -}; - -} // namespace cvcore - -#endif // CVCORE_CAMERAMODEL_H diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/core/ComputeEngine.h b/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/core/ComputeEngine.h deleted file mode 100644 index 65fe7ca..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/core/ComputeEngine.h +++ /dev/null @@ -1,43 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -#ifndef CVCORE_COMPUTEENGINE_H -#define CVCORE_COMPUTEENGINE_H - -#include - -namespace cvcore { - -enum class ComputeEngine : unsigned int -{ - UNKNOWN = 0x00, // 0000_0000 - - CPU = 0x01, // 0000_0001 - PVA = 0x02, // 0000_0010 - VIC = 0x04, // 0000_0100 - NVENC = 0x08, // 0000_1000 - GPU = 0x10, // 0001_0000 - DLA = 0x20, // 0010_0000 - DLA_CORE_0 = 0x40, // 0100_0000 - DLA_CORE_1 = 0x80, // 1000_0000 - - COMPUTE_FAULT = 0xFF // 1111_1111 -}; - -} // namespace cvcore - -#endif // CVCORE_COMPUTEENGINE_H diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/core/Core.h b/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/core/Core.h deleted file mode 100644 index 42732d9..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/core/Core.h +++ /dev/null @@ -1,35 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -#ifndef CORE_H -#define CORE_H - -namespace cvcore { - -// Enable dll imports/exports in case of windows support -#ifdef _WIN32 -#ifdef CVCORE_EXPORT_SYMBOLS // Needs to be enabled in case of compiling dll -#define CVCORE_API __declspec(dllexport) // Exports symbols when compiling the library. -#else -#define CVCORE_API __declspec(dllimport) // Imports the symbols when linked with library. -#endif -#else -#define CVCORE_API -#endif - -} // namespace cvcore -#endif // CORE_H diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/core/Image.h b/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/core/Image.h deleted file mode 100644 index 263a699..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/core/Image.h +++ /dev/null @@ -1,893 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2020-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -#ifndef CVCORE_IMAGE_H -#define CVCORE_IMAGE_H - -#include -#include -#include -#include - -#include "Memory.h" -#include "Tensor.h" - -namespace cvcore { - -/** - * An enum. - * Enum type for image type. - */ -enum ImageType -{ - Y_U8, /**< 8-bit unsigned gray. */ - Y_U16, /**< 16-bit unsigned gray. */ - Y_S8, /**< 8-bit signed gray. */ - Y_S16, /**< 16-bit signed gray. */ - Y_F16, /**< half normalized gray. */ - Y_F32, /**< float normalized gray. */ - RGB_U8, /**< 8-bit RGB. */ - RGB_U16, /**< 16-bit RGB. */ - RGB_F16, /**< half RGB. */ - RGB_F32, /**< float RGB. */ - BGR_U8, /**< 8-bit BGR. */ - BGR_U16, /**< 16-bit BGR. */ - BGR_F16, /**< half BGR. */ - BGR_F32, /**< float BGR. */ - RGBA_U8, /**< 8-bit RGBA. */ - RGBA_U16, /**< 16-bit RGBA. */ - RGBA_F16, /**< half RGBA. */ - RGBA_F32, /**< float RGBA. */ - PLANAR_RGB_U8, /**< 8-bit planar RGB. */ - PLANAR_RGB_U16, /**< 16-bit planar RGB. */ - PLANAR_RGB_F16, /**< half planar RGB. */ - PLANAR_RGB_F32, /**< float planar RGB. */ - PLANAR_BGR_U8, /**< 8-bit planar BGR. */ - PLANAR_BGR_U16, /**< 16-bit planar BGR. */ - PLANAR_BGR_F16, /**< half planar BGR. */ - PLANAR_BGR_F32, /**< float planar BGR. */ - PLANAR_RGBA_U8, /**< 8-bit planar RGBA. */ - PLANAR_RGBA_U16, /**< 16-bit planar RGBA. */ - PLANAR_RGBA_F16, /**< half planar RGBA. */ - PLANAR_RGBA_F32, /**< float planar RGBA. */ - NV12, /**< 8-bit planar Y + interleaved and subsampled (1/4 Y samples) UV. */ - NV24, /**< 8-bit planar Y + interleaved UV. */ -}; - -/** - * Struct type for image preprocessing params - */ -struct ImagePreProcessingParams -{ - ImageType imgType; /**< Input Image Type. */ - float pixelMean[3]; /**< Image Mean value offset for R,G,B channels. Default is 0.0f */ - float normalization[3]; /**< Scale or normalization values for R,G,B channels. Default is 1.0/255.0f */ - float stdDev[3]; /**< Standard deviation values for R,G,B channels. Default is 1.0f */ -}; - -template -struct IsCompositeImage : std::integral_constant -{ -}; - -template -struct IsPlanarImage - : std::integral_constant -{ -}; - -template -struct IsInterleavedImage : std::integral_constant::value && !IsPlanarImage::value> -{ -}; - -/** - * Image traits that map ImageType to TensorLayout, ChannelCount and ChannelType. - */ -template -struct ImageTraits; - -template<> -struct ImageTraits -{ - static constexpr TensorLayout TL = TensorLayout::HWC; - static constexpr ChannelCount CC = ChannelCount::C1; - static constexpr ChannelType CT = ChannelType::U8; -}; - -template<> -struct ImageTraits -{ - static constexpr TensorLayout TL = TensorLayout::NHWC; - static constexpr ChannelCount CC = ChannelCount::C1; - static constexpr ChannelType CT = ChannelType::U8; -}; - -template<> -struct ImageTraits -{ - static constexpr TensorLayout TL = TensorLayout::HWC; - static constexpr ChannelCount CC = ChannelCount::C1; - static constexpr ChannelType CT = ChannelType::U16; -}; - -template<> -struct ImageTraits -{ - static constexpr TensorLayout TL = TensorLayout::NHWC; - static constexpr ChannelCount CC = ChannelCount::C1; - static constexpr ChannelType CT = ChannelType::U16; -}; - -template<> -struct ImageTraits -{ - static constexpr TensorLayout TL = TensorLayout::HWC; - static constexpr ChannelCount CC = ChannelCount::C1; - static constexpr ChannelType CT = ChannelType::S8; -}; - -template<> -struct ImageTraits -{ - static constexpr TensorLayout TL = TensorLayout::NHWC; - static constexpr ChannelCount CC = ChannelCount::C1; - static constexpr ChannelType CT = ChannelType::S8; -}; - -template<> -struct ImageTraits -{ - static constexpr TensorLayout TL = TensorLayout::HWC; - static constexpr ChannelCount CC = ChannelCount::C1; - static constexpr ChannelType CT = ChannelType::S16; -}; - -template<> -struct ImageTraits -{ - static constexpr TensorLayout TL = TensorLayout::NHWC; - static constexpr ChannelCount CC = ChannelCount::C1; - static constexpr ChannelType CT = ChannelType::S16; -}; - -template<> -struct ImageTraits -{ - static constexpr TensorLayout TL = TensorLayout::HWC; - static constexpr ChannelCount CC = ChannelCount::C1; - static constexpr ChannelType CT = ChannelType::F32; -}; - -template<> -struct ImageTraits -{ - static constexpr TensorLayout TL = TensorLayout::NHWC; - static constexpr ChannelCount CC = ChannelCount::C1; - static constexpr ChannelType CT = ChannelType::F32; -}; - -template<> -struct ImageTraits -{ - static constexpr TensorLayout TL = TensorLayout::HWC; - static constexpr ChannelCount CC = ChannelCount::C3; - static constexpr ChannelType CT = ChannelType::U8; -}; - -template<> -struct ImageTraits -{ - static constexpr TensorLayout TL = TensorLayout::NHWC; - static constexpr ChannelCount CC = ChannelCount::C3; - static constexpr ChannelType CT = ChannelType::U8; -}; - -template<> -struct ImageTraits -{ - static constexpr TensorLayout TL = TensorLayout::HWC; - static constexpr ChannelCount CC = ChannelCount::C3; - static constexpr ChannelType CT = ChannelType::U16; -}; - -template<> -struct ImageTraits -{ - static constexpr TensorLayout TL = TensorLayout::NHWC; - static constexpr ChannelCount CC = ChannelCount::C3; - static constexpr ChannelType CT = ChannelType::U16; -}; - -template<> -struct ImageTraits -{ - static constexpr TensorLayout TL = TensorLayout::HWC; - static constexpr ChannelCount CC = ChannelCount::C3; - static constexpr ChannelType CT = ChannelType::F32; -}; - -template<> -struct ImageTraits -{ - static constexpr TensorLayout TL = TensorLayout::NHWC; - static constexpr ChannelCount CC = ChannelCount::C3; - static constexpr ChannelType CT = ChannelType::F32; -}; - -template<> -struct ImageTraits -{ - static constexpr TensorLayout TL = TensorLayout::HWC; - static constexpr ChannelCount CC = ChannelCount::C3; - static constexpr ChannelType CT = ChannelType::U8; -}; - -template<> -struct ImageTraits -{ - static constexpr TensorLayout TL = TensorLayout::NHWC; - static constexpr ChannelCount CC = ChannelCount::C3; - static constexpr ChannelType CT = ChannelType::U8; -}; - -template<> -struct ImageTraits -{ - static constexpr TensorLayout TL = TensorLayout::HWC; - static constexpr ChannelCount CC = ChannelCount::C3; - static constexpr ChannelType CT = ChannelType::U16; -}; - -template<> -struct ImageTraits -{ - static constexpr TensorLayout TL = TensorLayout::NHWC; - static constexpr ChannelCount CC = ChannelCount::C3; - static constexpr ChannelType CT = ChannelType::U16; -}; - -template<> -struct ImageTraits -{ - static constexpr TensorLayout TL = TensorLayout::HWC; - static constexpr ChannelCount CC = ChannelCount::C3; - static constexpr ChannelType CT = ChannelType::F32; -}; - -template<> -struct ImageTraits -{ - static constexpr TensorLayout TL = TensorLayout::NHWC; - static constexpr ChannelCount CC = ChannelCount::C3; - static constexpr ChannelType CT = ChannelType::F32; -}; - -template<> -struct ImageTraits -{ - static constexpr TensorLayout TL = TensorLayout::CHW; - static constexpr ChannelCount CC = ChannelCount::C3; - static constexpr ChannelType CT = ChannelType::U8; -}; - -template<> -struct ImageTraits -{ - static constexpr TensorLayout TL = TensorLayout::NCHW; - static constexpr ChannelCount CC = ChannelCount::C3; - static constexpr ChannelType CT = ChannelType::U8; -}; - -template<> -struct ImageTraits -{ - static constexpr TensorLayout TL = TensorLayout::CHW; - static constexpr ChannelCount CC = ChannelCount::C3; - static constexpr ChannelType CT = ChannelType::U16; -}; - -template<> -struct ImageTraits -{ - static constexpr TensorLayout TL = TensorLayout::NCHW; - static constexpr ChannelCount CC = ChannelCount::C3; - static constexpr ChannelType CT = ChannelType::U16; -}; - -template<> -struct ImageTraits -{ - static constexpr TensorLayout TL = TensorLayout::CHW; - static constexpr ChannelCount CC = ChannelCount::C3; - static constexpr ChannelType CT = ChannelType::F32; -}; - -template<> -struct ImageTraits -{ - static constexpr TensorLayout TL = TensorLayout::NCHW; - static constexpr ChannelCount CC = ChannelCount::C3; - static constexpr ChannelType CT = ChannelType::F32; -}; - -template<> -struct ImageTraits -{ - static constexpr TensorLayout TL = TensorLayout::CHW; - static constexpr ChannelCount CC = ChannelCount::C3; - static constexpr ChannelType CT = ChannelType::U8; -}; - -template<> -struct ImageTraits -{ - static constexpr TensorLayout TL = TensorLayout::NCHW; - static constexpr ChannelCount CC = ChannelCount::C3; - static constexpr ChannelType CT = ChannelType::U8; -}; - -template<> -struct ImageTraits -{ - static constexpr TensorLayout TL = TensorLayout::CHW; - static constexpr ChannelCount CC = ChannelCount::C3; - static constexpr ChannelType CT = ChannelType::U16; -}; - -template<> -struct ImageTraits -{ - static constexpr TensorLayout TL = TensorLayout::NCHW; - static constexpr ChannelCount CC = ChannelCount::C3; - static constexpr ChannelType CT = ChannelType::U16; -}; - -template<> -struct ImageTraits -{ - static constexpr TensorLayout TL = TensorLayout::CHW; - static constexpr ChannelCount CC = ChannelCount::C3; - static constexpr ChannelType CT = ChannelType::F32; -}; - -template<> -struct ImageTraits -{ - static constexpr TensorLayout TL = TensorLayout::NCHW; - static constexpr ChannelCount CC = ChannelCount::C3; - static constexpr ChannelType CT = ChannelType::F32; -}; - -/** - * Get the bytes of each element for a specific ImageType. - */ -inline size_t GetImageElementSize(const ImageType type) -{ - size_t imageElementSize; - - switch (type) - { - case ImageType::Y_U8: - case ImageType::Y_S8: - case ImageType::RGB_U8: - case ImageType::BGR_U8: - case ImageType::RGBA_U8: - case ImageType::PLANAR_RGB_U8: - case ImageType::PLANAR_BGR_U8: - case ImageType::PLANAR_RGBA_U8: - { - imageElementSize = 1; - break; - } - case ImageType::Y_U16: - case ImageType::Y_S16: - case ImageType::RGB_U16: - case ImageType::BGR_U16: - case ImageType::RGBA_U16: - case ImageType::PLANAR_RGB_U16: - case ImageType::PLANAR_BGR_U16: - case ImageType::PLANAR_RGBA_U16: - case ImageType::Y_F16: - case ImageType::RGB_F16: - case ImageType::BGR_F16: - case ImageType::RGBA_F16: - case ImageType::PLANAR_RGB_F16: - case ImageType::PLANAR_BGR_F16: - case ImageType::PLANAR_RGBA_F16: - { - imageElementSize = 2; - break; - } - case ImageType::Y_F32: - case ImageType::RGB_F32: - case ImageType::BGR_F32: - case ImageType::RGBA_F32: - case ImageType::PLANAR_RGB_F32: - case ImageType::PLANAR_BGR_F32: - case ImageType::PLANAR_RGBA_F32: - { - imageElementSize = 4; - break; - } - default: - { - imageElementSize = 0; - } - } - - return imageElementSize; -} - -/** - * Get the number of channels for a specific ImageType. - */ -inline size_t GetImageChannelCount(const ImageType type) -{ - size_t imageChannelCount; - - switch (type) - { - case ImageType::Y_U8: - case ImageType::Y_U16: - case ImageType::Y_S8: - case ImageType::Y_S16: - case ImageType::Y_F16: - case ImageType::Y_F32: - { - imageChannelCount = 1; - break; - } - case ImageType::RGB_U8: - case ImageType::RGB_U16: - case ImageType::RGB_F16: - case ImageType::RGB_F32: - case ImageType::BGR_U8: - case ImageType::BGR_U16: - case ImageType::BGR_F16: - case ImageType::BGR_F32: - case ImageType::PLANAR_RGB_U8: - case ImageType::PLANAR_RGB_U16: - case ImageType::PLANAR_RGB_F16: - case ImageType::PLANAR_RGB_F32: - case ImageType::PLANAR_BGR_U8: - case ImageType::PLANAR_BGR_U16: - case ImageType::PLANAR_BGR_F16: - case ImageType::PLANAR_BGR_F32: - { - imageChannelCount = 3; - break; - } - case ImageType::RGBA_U8: - case ImageType::RGBA_U16: - case ImageType::RGBA_F16: - case ImageType::RGBA_F32: - case ImageType::PLANAR_RGBA_U8: - case ImageType::PLANAR_RGBA_U16: - case ImageType::PLANAR_RGBA_F16: - case ImageType::PLANAR_RGBA_F32: - { - imageChannelCount = 4; - break; - } - default: - { - imageChannelCount = 0; - } - } - - return imageChannelCount; -}; - -template -class Image -{ -}; - -template<> -class Image : public Tensor -{ - using Tensor::Tensor; -}; - -template<> -class Image : public Tensor -{ - using Tensor::Tensor; -}; - -template<> -class Image : public Tensor -{ - using Tensor::Tensor; -}; - -template<> -class Image : public Tensor -{ - using Tensor::Tensor; -}; - -template<> -class Image : public Tensor -{ - using Tensor::Tensor; -}; - -template<> -class Image : public Tensor -{ - using Tensor::Tensor; -}; - -template<> -class Image : public Tensor -{ - using Tensor::Tensor; -}; - -template<> -class Image : public Tensor -{ - using Tensor::Tensor; -}; - -template<> -class Image : public Tensor -{ - using Tensor::Tensor; -}; - -template<> -class Image : public Tensor -{ - using Tensor::Tensor; -}; - -template<> -class Image : public Tensor -{ - using Tensor::Tensor; -}; - -template<> -class Image : public Tensor -{ - using Tensor::Tensor; -}; - -template<> -class Image : public Tensor -{ - using Tensor::Tensor; -}; - -template<> -class Image : public Tensor -{ - using Tensor::Tensor; -}; - -template<> -class Image : public Tensor -{ - using Tensor::Tensor; -}; - -template<> -class Image : public Tensor -{ - using Tensor::Tensor; -}; - -template<> -class Image : public Tensor -{ - using Tensor::Tensor; -}; - -template<> -class Image : public Tensor -{ - using Tensor::Tensor; -}; - -template<> -class Image : public Tensor -{ - using Tensor::Tensor; -}; - -template<> -class Image : public Tensor -{ - using Tensor::Tensor; -}; - -template<> -class Image : public Tensor -{ - using Tensor::Tensor; -}; - -template<> -class Image : public Tensor -{ - using Tensor::Tensor; -}; - -template<> -class Image : public Tensor -{ - using Tensor::Tensor; -}; - -template<> -class Image : public Tensor -{ - using Tensor::Tensor; -}; - -template<> -class Image : public Tensor -{ - using Tensor::Tensor; -}; - -template<> -class Image : public Tensor -{ - using Tensor::Tensor; -}; - -template<> -class Image : public Tensor -{ - using Tensor::Tensor; -}; - -template<> -class Image : public Tensor -{ - using Tensor::Tensor; -}; - -template<> -class Image : public Tensor -{ - using Tensor::Tensor; -}; - -template<> -class Image : public Tensor -{ - using Tensor::Tensor; -}; - -template<> -class Image -{ -public: - Image(std::size_t width, std::size_t height, bool isCPU = true) - : m_data(std::make_tuple(Y(width, height, isCPU), UV(width / 2, height / 2, isCPU))) - { - assert(width % 2 == 0 && height % 2 == 0); - } - - Image(std::size_t width, std::size_t height, std::uint8_t *dataPtrLuma, std::uint8_t *dataPtrChroma, - bool isCPU = true) - : m_data(std::make_tuple(Y(width, height, dataPtrLuma, isCPU), UV(width / 2, height / 2, dataPtrChroma, isCPU))) - { - assert(width % 2 == 0 && height % 2 == 0); - } - - Image(std::size_t width, std::size_t height, std::size_t rowPitchLuma, std::size_t rowPitchChroma, - std::uint8_t *dataPtrLuma, std::uint8_t *dataPtrChroma, bool isCPU = true) - : m_data(std::make_tuple(Y(width, height, rowPitchLuma, dataPtrLuma, isCPU), - UV(width / 2, height / 2, rowPitchChroma, dataPtrChroma, isCPU))) - { - assert(width % 2 == 0 && height % 2 == 0); - } - - std::size_t getLumaWidth() const - { - return std::get<0>(m_data).getWidth(); - } - - std::size_t getLumaHeight() const - { - return std::get<0>(m_data).getHeight(); - } - - std::size_t getChromaWidth() const - { - return std::get<1>(m_data).getWidth(); - } - - std::size_t getChromaHeight() const - { - return std::get<1>(m_data).getHeight(); - } - - std::size_t getLumaStride(TensorDimension dim) const - { - return std::get<0>(m_data).getStride(dim); - } - - std::size_t getChromaStride(TensorDimension dim) const - { - return std::get<1>(m_data).getStride(dim); - } - - std::uint8_t *getLumaData() - { - return std::get<0>(m_data).getData(); - } - - std::uint8_t *getChromaData() - { - return std::get<1>(m_data).getData(); - } - - const std::uint8_t *getLumaData() const - { - return std::get<0>(m_data).getData(); - } - - std::size_t getLumaDataSize() const - { - return std::get<0>(m_data).getDataSize(); - } - - const std::uint8_t *getChromaData() const - { - return std::get<1>(m_data).getData(); - } - - std::size_t getChromaDataSize() const - { - return std::get<1>(m_data).getDataSize(); - } - - bool isCPU() const - { - return std::get<0>(m_data).isCPU(); - } - - friend void Copy(Image &dst, const Image &src, cudaStream_t stream); - -private: - using Y = Tensor; - using UV = Tensor; - - std::tuple m_data; -}; - -template<> -class Image -{ -public: - Image(std::size_t width, std::size_t height, bool isCPU = true) - : m_data(std::make_tuple(Y(width, height, isCPU), UV(width, height, isCPU))) - { - } - - Image(std::size_t width, std::size_t height, std::uint8_t *dataPtrLuma, std::uint8_t *dataPtrChroma, - bool isCPU = true) - : m_data(std::make_tuple(Y(width, height, dataPtrLuma, isCPU), UV(width, height, dataPtrChroma, isCPU))) - { - } - - Image(std::size_t width, std::size_t height, std::size_t rowPitchLuma, std::size_t rowPitchChroma, - std::uint8_t *dataPtrLuma, std::uint8_t *dataPtrChroma, bool isCPU = true) - : m_data(std::make_tuple(Y(width, height, rowPitchLuma, dataPtrLuma, isCPU), - UV(width, height, rowPitchChroma, dataPtrChroma, isCPU))) - { - } - - std::size_t getLumaWidth() const - { - return std::get<0>(m_data).getWidth(); - } - - std::size_t getLumaHeight() const - { - return std::get<0>(m_data).getHeight(); - } - - std::size_t getChromaWidth() const - { - return std::get<1>(m_data).getWidth(); - } - - std::size_t getChromaHeight() const - { - return std::get<1>(m_data).getHeight(); - } - - std::size_t getLumaStride(TensorDimension dim) const - { - return std::get<0>(m_data).getStride(dim); - } - - std::size_t getChromaStride(TensorDimension dim) const - { - return std::get<1>(m_data).getStride(dim); - } - - std::uint8_t *getLumaData() - { - return std::get<0>(m_data).getData(); - } - - const std::uint8_t *getLumaData() const - { - return std::get<0>(m_data).getData(); - } - - std::size_t getLumaDataSize() const - { - return std::get<0>(m_data).getDataSize(); - } - - std::uint8_t *getChromaData() - { - return std::get<1>(m_data).getData(); - } - - const std::uint8_t *getChromaData() const - { - return std::get<1>(m_data).getData(); - } - - std::size_t getChromaDataSize() const - { - return std::get<1>(m_data).getDataSize(); - } - - bool isCPU() const - { - return std::get<0>(m_data).isCPU(); - } - - friend void Copy(Image &dst, const Image &src, cudaStream_t stream); - -private: - using Y = Tensor; - using UV = Tensor; - - std::tuple m_data; -}; - -void inline Copy(Image &dst, const Image &src, cudaStream_t stream = 0) -{ - Copy(std::get<0>(dst.m_data), std::get<0>(src.m_data), stream); - Copy(std::get<1>(dst.m_data), std::get<1>(src.m_data), stream); -} - -void inline Copy(Image &dst, const Image &src, cudaStream_t stream = 0) -{ - Copy(std::get<0>(dst.m_data), std::get<0>(src.m_data), stream); - Copy(std::get<1>(dst.m_data), std::get<1>(src.m_data), stream); -} - -} // namespace cvcore - -#endif // CVCORE_IMAGE_H diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/core/Instrumentation.h b/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/core/Instrumentation.h deleted file mode 100644 index 6324b8d..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/core/Instrumentation.h +++ /dev/null @@ -1,65 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -#ifndef CVCORE_INSTRUMENTATION_H -#define CVCORE_INSTRUMENTATION_H - -#include - -namespace cvcore { namespace profiler { - -/** - * A enum class used to find out the type of profiler output required - */ -enum class ProfilerJsonOutputType : uint32_t -{ - JSON_OFF, /**< print the aggregate values of each timer in pretty print format */ - JSON_AGGREGATE, /**< print the aggregate values of each timer in JSON format - along with the pretty print format. Pretty print format - gets printed on the terminal */ - JSON_SEPARATE /**< print all the elapsed times for all timers along with the - aggregate values from JSON_AGGREGATE option */ -}; - -/** -* Flush call to print the timer values in a file input -* @param jsonHelperType used to find out the type of profiler output required -* @return filename used to write the timer values -*/ -void flush(const std::string& filename, ProfilerJsonOutputType jsonHelperType); - -/** -* Flush call to print the timer values in a output stream -* @param jsonHelperType used to find out the type of profiler output required -* @return output stream used to write the timer values -*/ -void flush(std::ostream& output, ProfilerJsonOutputType jsonHelperType); - -/** -* Flush call to print the timer values on the terminal -* @param jsonHelperType used to find out the type of profiler output required -*/ -void flush(ProfilerJsonOutputType jsonHelperType); - - -/** -* Clear all the profile timers -*/ -void clear(); - -}} -#endif diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/core/MathTypes.h b/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/core/MathTypes.h deleted file mode 100644 index fd9db1a..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/core/MathTypes.h +++ /dev/null @@ -1,234 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2020-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -#ifndef CVCORE_Math_H -#define CVCORE_Math_H - -#include -#include -#include -#include - -#include "Tensor.h" - -namespace cvcore { - -/** Matrix types using Tensor backend */ -using Matrixf = Tensor; -using Matrixd = Tensor; - -/** - * A struct. - * Structure used to store Vector2 Values. - */ -template -struct Vector2 -{ - T x; /**< point x coordinate. */ - T y; /**< point y coordinate. */ - - inline T &operator[](size_t i) - { - if (i == 0) - { - return x; - } - else if (i == 1) - { - return y; - } - else - { - throw std::out_of_range("cvcore::Vector2 ==> Requested index is out of bounds"); - } - } -}; - -/** - * A struct. - * Structure used to store Vector3 Values. - */ -template -struct Vector3 -{ - T x; /**< point x coordinate. */ - T y; /**< point y coordinate. */ - T z; /**< point z coordinate. */ - - inline T &operator[](size_t i) - { - if (i == 0) - { - return x; - } - else if (i == 1) - { - return y; - } - else if (i == 2) - { - return z; - } - else - { - throw std::out_of_range("cvcore::Vector3 ==> Requested index is out of bounds"); - } - } -}; - -using Vector2i = Vector2; -using Vector3i = Vector3; - -using Vector2f = Vector2; -using Vector3f = Vector3; - -using Vector2d = Vector2; -using Vector3d = Vector3; - -/** - * A struct - * Structure used to store AxisAngle Rotation parameters. - */ -struct AxisAngleRotation -{ - double angle; /** Counterclockwise rotation angle [0, 2PI]. */ - Vector3d axis; /** 3d axis of rotation. */ - - AxisAngleRotation() - : angle(0.0) - , axis{0, 0, 0} - { - } - - AxisAngleRotation(double angleinput, Vector3d axisinput) - : angle(angleinput) - , axis(axisinput) - { - } -}; - -/** - * A struct. - * Structure used to store quaternion rotation representation. - * A rotation of unit vector u with rotation theta can be represented in quaternion as: - * q={cos(theta/2)+ i(u*sin(theta/2))} -*/ -struct Quaternion -{ - double qx, qy, qz; /** Axis or imaginary component of the quaternion representation. */ - double qw; /** Angle or real component of the quaternion representation. */ - - Quaternion() - : qx(0.0) - , qy(0.0) - , qz(0.0) - , qw(0.0) - { - } - - Quaternion(double qxinput, double qyinput, double qzinput, double qwinput) - : qx(qxinput) - , qy(qyinput) - , qz(qzinput) - , qw(qwinput) - { - } -}; - -/** - * Convert rotation matrix to rotation vector. - * @param rotMatrix Rotation matrix of 3x3 values. - * @return 3D Rotation vector {theta * xaxis, theta * yaxis, theta * zaxis} - * where theta is the angle of rotation in radians - */ -Vector3d RotationMatrixToRotationVector(const std::vector &rotMatrix); - -/** - * Convert rotation matrix to axis angle representation. - * @param rotMatrix Rotation matrix of 3x3 values. - * @return Axis angle rotation - */ -AxisAngleRotation RotationMatrixToAxisAngleRotation(const std::vector &rotMatrix); - -/** - * Convert axis angle representation to rotation matrix. - * @param axisangle Axis angle rotation. - * @return Rotation matrix of 3x3 values. - */ -std::vector AxisAngleToRotationMatrix(const AxisAngleRotation &axisangle); - -/** - * Convert axis angle representation to 3d rotation vector. - * Rotation vector is {theta * xaxis, theta * yaxis, theta * zaxis} - * where theta is the angle of rotation in radians. - * @param axisangle Axis angle rotation. - * @return 3D Rotation Vector - */ -Vector3d AxisAngleRotationToRotationVector(const AxisAngleRotation &axisangle); - -/** - * Convert rotation vector to axis angle representation. - * @param rotVector 3D rotation vector. - * @return Axis angle rotation. - */ -AxisAngleRotation RotationVectorToAxisAngleRotation(const Vector3d &rotVector); - -/** - * Convert axis angle representation to quaternion. - * @param axisangle Axis angle representation. - * @return Quaternion rotation. - */ -Quaternion AxisAngleRotationToQuaternion(const AxisAngleRotation &axisangle); - -/** - * Convert quaternion rotation to axis angle rotation. - * @param qrotation Quaternion rotation representation. - * @return Axis angle rotation. - */ -AxisAngleRotation QuaternionToAxisAngleRotation(const Quaternion &qrotation); - -/** - * Convert quaternion rotation to rotation matrix. - * @param qrotation Quaternion rotation representation. - * @return Rotation matrix. - */ -std::vector QuaternionToRotationMatrix(const Quaternion &qrotation); - -/** - * Convert rotation matrix to Quaternion. - * @param rotMatrix Rotation matrix - * @return Quaternion rotation. - */ -Quaternion RotationMatrixToQuaternion(const std::vector &rotMatrix); - -/** - * A struct. - * Structure used to store Pose3D parameters. - */ -template -struct Pose3 -{ - AxisAngleRotation rotation; /**Rotation expressed in axis angle notation.*/ - Vector3 translation; /*Translation expressed as x,y,z coordinates.*/ -}; - -using Pose3d = Pose3; -using Pose3f = Pose3; - -} // namespace cvcore - -#endif // CVCORE_Math_H diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/core/Memory.h b/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/core/Memory.h deleted file mode 100644 index 7d3d113..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/core/Memory.h +++ /dev/null @@ -1,135 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2020-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -#ifndef CVCORE_MEMORY_H -#define CVCORE_MEMORY_H - -#include - -#include "Tensor.h" - -namespace cvcore { - -/** - * Implementation of tensor copy. - * @param dst destination TensorBase. - * @param src source TensorBase. - * @param stream cuda stream. - */ -void TensorBaseCopy(TensorBase &dst, const TensorBase &src, cudaStream_t stream = 0); - -/** - * Implementation of tensor copy for 2D pitch linear tensors. - * @param dst destination TensorBase. - * @param src source TensorBase. - * @param dstPitch pitch of destination Tensor in bytes. - * @param srcPitch pitch of source Tensor in bytes. - * @param widthInBytes width in bytes. - * @param height height of tensor. - * @param stream cuda stream. - */ -void TensorBaseCopy2D(TensorBase &dst, const TensorBase &src, int dstPitch, int srcPitch, int widthInBytes, int height, - cudaStream_t stream = 0); - -/** - * Memory copy function between two non HWC/CHW/NHWC/NCHW Tensors. - * @tparam TL TensorLayout type. - * @tparam CC Channel Count. - * @tparam CT ChannelType. - * @param dst destination Tensor. - * @param src source Tensor which copy from. - * @param stream cuda stream. - */ -template::type * = nullptr> -void Copy(Tensor &dst, const Tensor &src, cudaStream_t stream = 0) -{ - TensorBaseCopy(dst, src, stream); -} - -/** - * Memory copy function between two HWC Tensors. - * @tparam TL TensorLayout type. - * @tparam CC Channel Count. - * @tparam CT ChannelType. - * @param dst destination Tensor. - * @param src source Tensor which copy from. - * @param stream cuda stream. - */ -template::type * = nullptr> -void Copy(Tensor &dst, const Tensor &src, cudaStream_t stream = 0) -{ - TensorBaseCopy2D(dst, src, dst.getStride(TensorDimension::HEIGHT) * GetChannelSize(CT), - src.getStride(TensorDimension::HEIGHT) * GetChannelSize(CT), - dst.getWidth() * dst.getChannelCount() * GetChannelSize(CT), src.getHeight(), stream); -} - -/** - * Memory copy function between two NHWC Tensors. - * @tparam TL TensorLayout type. - * @tparam CC Channel Count. - * @tparam CT ChannelType. - * @param dst destination Tensor. - * @param src source Tensor which copy from. - * @param stream cuda stream. - */ -template::type * = nullptr> -void Copy(Tensor &dst, const Tensor &src, cudaStream_t stream = 0) -{ - TensorBaseCopy2D(dst, src, dst.getStride(TensorDimension::HEIGHT) * GetChannelSize(CT), - src.getStride(TensorDimension::HEIGHT) * GetChannelSize(CT), - dst.getWidth() * dst.getChannelCount() * GetChannelSize(CT), src.getDepth() * src.getHeight(), - stream); -} - -/** - * Memory copy function between two CHW Tensors. - * @tparam TL TensorLayout type. - * @tparam CC Channel Count. - * @tparam CT ChannelType. - * @param dst destination Tensor. - * @param src source Tensor which copy from. - * @param stream cuda stream. - */ -template::type * = nullptr> -void Copy(Tensor &dst, const Tensor &src, cudaStream_t stream = 0) -{ - TensorBaseCopy2D(dst, src, dst.getStride(TensorDimension::HEIGHT) * GetChannelSize(CT), - src.getStride(TensorDimension::HEIGHT) * GetChannelSize(CT), dst.getWidth() * GetChannelSize(CT), - src.getChannelCount() * src.getHeight(), stream); -} - -/** - * Memory copy function between two NCHW Tensors. - * @tparam TL TensorLayout type. - * @tparam CC Channel Count. - * @tparam CT ChannelType. - * @param dst destination Tensor. - * @param src source Tensor which copy from. - * @param stream cuda stream. - */ -template::type * = nullptr> -void Copy(Tensor &dst, const Tensor &src, cudaStream_t stream = 0) -{ - TensorBaseCopy2D(dst, src, dst.getStride(TensorDimension::HEIGHT) * GetChannelSize(CT), - src.getStride(TensorDimension::HEIGHT) * GetChannelSize(CT), dst.getWidth() * GetChannelSize(CT), - src.getDepth() * src.getChannelCount() * src.getHeight(), stream); -} - -} // namespace cvcore - -#endif // CVCORE_MEMORY_H diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/core/ProfileUtils.h b/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/core/ProfileUtils.h deleted file mode 100644 index 0a4e9a5..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/core/ProfileUtils.h +++ /dev/null @@ -1,40 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2020-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -#ifndef CVCORE_PROFILE_UTILS_H -#define CVCORE_PROFILE_UTILS_H - -#include - -namespace cvcore { - -/** - * Export one profiling item to specified json file. - * @param outputPath output json file path. - * @param taskName item name showing in the output json file. - * @param tMin minimum running time in milliseconds. - * @param tMax maximum running time in milliseconds. - * @param tAvg average running time in milliseconds. - * @param isCPU whether CPU or GPU time. - * @param iterations number of iterations. - */ -void ExportToJson(const std::string outputPath, const std::string taskName, float tMin, float tMax, float tAvg, - bool isCPU, int iterations = 100); - -} // namespace cvcore - -#endif // CVCORE_PROFILE_UTILS_H diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/core/Tensor.h b/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/core/Tensor.h deleted file mode 100644 index b06e531..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/core/Tensor.h +++ /dev/null @@ -1,1189 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2019-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -#ifndef CVCORE_TENSOR_H -#define CVCORE_TENSOR_H - -#include -#include -#include -#include - -namespace cvcore { - -// there is no CUDA dependency at the data level, so we map half type to uint16_t for now -using half = std::uint16_t; - -/** - * An enum. - * Enum type for tensor layout type. - */ -enum TensorLayout -{ - LC, /**< length, channel (channel interleaved). */ - CL, /**< channel, length (channel planar). */ - HWC, /**< height, width, channel (channel interleaved). */ - CHW, /**< channel, height, width (channel planar). */ - DHWC, /**< depth, height, width, channel (channel interleaved). */ - NHWC = DHWC, /**< alias for DHWC. */ - DCHW, /**< depth, channel, height, width (channel planar). */ - NCHW = DCHW, /**< alias for DCHW. */ - CDHW, /**< channel, depth, height, width (channel planar). */ -}; - -/** - * An enum. - * Enum type for tensor channel count. - */ -enum ChannelCount -{ - C1, /**< 1 channels. */ - C2, /**< 2 channels. */ - C3, /**< 3 channels. */ - C4, /**< 4 channels. */ - CX, /**< varying number of channels. */ -}; - -/** - * An enum. - * Enum type for channel type. - */ -enum ChannelType -{ - U8, /**< uint8_t. */ - U16, /**< uint16_t. */ - S8, /**< int8_t. */ - S16, /**< int16_t. */ - F16, /**< cvcore::half. */ - F32, /**< float. */ - F64, /**< double. */ -}; - -/** - * An enum. - * Enum type for dimension type. - */ -enum class TensorDimension -{ - LENGTH, /**< length dimension. */ - HEIGHT, /**< height dimension. */ - WIDTH, /**< width dimension. */ - CHANNEL, /**< channel dimension. */ - DEPTH, /**< depth dimension. */ -}; - -/** - * Function to get name of a TensorLayout value as string. - * @param TL the TensorLayout value. - * @return string name of TL. - */ -std::string GetTensorLayoutAsString(TensorLayout TL); - -/** - * Function to get name of a ChannelCount value as string. - * @param CC the ChannelCount value. - * @return string name of CC. - */ -std::string GetChannelCountAsString(ChannelCount CC); - -/** - * Function to get name of a ChannelType value as string. - * @param CT the ChannelType value. - * @return string name of CT. - */ -std::string GetChannelTypeAsString(ChannelType CT); - -/** - * Function to get name of a Memory type used. - * @param bool isCPU of Tensor as input - * @return string name of Memory type. - */ -std::string GetMemoryTypeAsString(bool isCPU); - -/** - * Function to get element size (in bytes) of a ChannelType. - * @param CT the ChannelType value. - * @return size in bytes. - */ -std::size_t GetChannelSize(ChannelType CT); - -/** - * Implementation of TensorBase class. - */ -class TensorBase -{ -public: - /** - * Struct for storing dimension data. - */ - struct DimData - { - std::size_t size; /**< size of each dimension. */ - std::size_t stride; /**< stride of each dimension. */ - }; - - /** - * Constructor of a non-owning tensor. - * @param type ChannelType of the Tensor. - * @param dimData pointer to the DimData array. - * @param dimCount number of dimensions. - * @param dataPtr raw pointer to the source data array. - * @param isCPU whether to allocate tensor on CPU or GPU. - */ - TensorBase(ChannelType type, const DimData *dimData, int dimCount, void *dataPtr, bool isCPU); - - /** - * Constructor of a non-owning tensor. - * @param type ChannelType of the Tensor. - * @param dimData initializer_list of DimData. - * @param dataPtr raw pointer to the source data array. - * @param isCPU whether to allocate tensor on CPU or GPU. - */ - TensorBase(ChannelType type, std::initializer_list dimData, void *dataPtr, bool isCPU); - - /** - * Constructor of a memory-owning tensor. - * @param type ChannelType of the Tensor. - * @param dimData pointer to the DimData array. - * @param dimCount number of dimensions. - * @param isCPU whether to allocate tensor on CPU or GPU. - */ - TensorBase(ChannelType type, const DimData *dimData, int dimCount, bool isCPU); - - /** - * Constructor of a memory-owning tensor. - * @param type ChannelType of the Tensor. - * @param dimData initializer_list of DimData. - * @param isCPU whether to allocate tensor on CPU or GPU. - */ - TensorBase(ChannelType type, std::initializer_list dimData, bool isCPU); - - /** - * Destructor of Tensor. - */ - ~TensorBase(); - - /** - * TensorBase is non-copyable. - */ - TensorBase(const TensorBase &) = delete; - - /** - * TensorBase is non-copyable. - */ - TensorBase &operator=(const TensorBase &) = delete; - - /** - * Move Constructor of TensorBase. - */ - TensorBase(TensorBase &&); - - /** - * Move operator of TensorBase. - */ - TensorBase &operator=(TensorBase &&); - - /** - * Get the dimension count of TensorBase. - * @return number of dimensions. - */ - int getDimCount() const; - - /** - * Get the size of given dimension. - * @param dimIdx dimension index. - * @return size of the specified dimension. - */ - std::size_t getSize(int dimIdx) const; - - /** - * Get the stride of given dimension. - * @param dimIdx dimension index. - * @return stride of the specified dimension. - */ - std::size_t getStride(int dimIdx) const; - - /** - * Get the ChannelType of the Tensor. - * @return ChannelType of the Tensor. - */ - ChannelType getType() const; - - /** - * Get the raw data pointer to the Tensor. - * @return void data pointer to the Tensor. - */ - void *getData() const; - - /** - * Get the total size of the Tensor in bytes. - * @return total bytes of the Tensor. - */ - std::size_t getDataSize() const; - - /** - * Get the flag whether the Tensor is allocated in CPU or GPU. - * @return whether the Tensor is allocated in CPU. - */ - bool isCPU() const; - - /** - * Get the flag whether the Tensor owns the data. - * @return whether the Tensor owns data in memory. - */ - bool isOwning() const; - -protected: - TensorBase(); - -private: - static constexpr int kMinDimCount = 2; - static constexpr int kMaxDimCount = 4; - - void *m_data; - - int m_dimCount; - DimData m_dimData[kMaxDimCount]; - - ChannelType m_type; - bool m_isOwning; - bool m_isCPU; -}; - -namespace detail { - -template -struct DimToIndex2D -{ - static_assert(TL == LC || TL == CL, "unsupported variant!"); - static constexpr int kLength = TL == LC ? 0 : 1; - static constexpr int kChannel = TL == LC ? 1 : 0; -}; - -template -struct DimToIndex3D -{ - static_assert(TL == HWC || TL == CHW, "unsupported variant!"); - static constexpr int kWidth = TL == HWC ? 1 : 2; - static constexpr int kHeight = TL == HWC ? 0 : 1; - static constexpr int kChannel = TL == HWC ? 2 : 0; -}; - -template -struct DimToIndex4D -{ - static_assert(TL == DHWC || TL == DCHW || TL == CDHW, "unsupported variant!"); - static constexpr int kWidth = TL == DHWC ? 2 : (TL == DCHW ? 3 : 3); - static constexpr int kHeight = TL == DHWC ? 1 : (TL == DCHW ? 2 : 2); - static constexpr int kDepth = TL == DHWC ? 0 : (TL == DCHW ? 0 : 1); - static constexpr int kChannel = TL == DHWC ? 3 : (TL == DCHW ? 1 : 0); -}; - -template -struct LayoutToIndex -{ -}; - -template -struct LayoutToIndex::type> : public DimToIndex2D -{ - static constexpr int kDimCount = 2; -}; - -template -struct LayoutToIndex::type> : public DimToIndex3D -{ - static constexpr int kDimCount = 3; -}; - -template -struct LayoutToIndex::type> - : public DimToIndex4D -{ - static constexpr int kDimCount = 4; -}; - -template -struct ChannelTypeToNative -{ -}; - -template<> -struct ChannelTypeToNative -{ - using Type = std::uint8_t; -}; - -template<> -struct ChannelTypeToNative -{ - using Type = std::uint16_t; -}; - -template<> -struct ChannelTypeToNative -{ - using Type = std::int8_t; -}; - -template<> -struct ChannelTypeToNative -{ - using Type = std::int16_t; -}; - -template<> -struct ChannelTypeToNative -{ - using Type = float; -}; - -template<> -struct ChannelTypeToNative -{ - using Type = cvcore::half; -}; - -template<> -struct ChannelTypeToNative -{ - using Type = double; -}; - -template -constexpr std::size_t ChannelToCount() -{ - switch (CC) - { - case C1: - return 1; - case C2: - return 2; - case C3: - return 3; - case C4: - return 4; - } - return 0; // this is safe as this function will never be called for dynamic channel counts -} - -/** - * Implementation of 2D tensors. - * @tparam TL tensor layout type. - * @tparam CT channel type. - */ -template -class Tensor2D : public TensorBase -{ - using DataType = typename ChannelTypeToNative::Type; - -public: - /** - * Default Constructor. - */ - Tensor2D() = default; - - /** - * Constructor of a memory-owning 2D tensor. - * @param dimData initializer_list of DimData. - * @param isCPU whether to allocate tensor on CPU or GPU. - */ - Tensor2D(std::initializer_list dimData, bool isCPU) - : TensorBase(CT, dimData, isCPU) - { - } - - /** - * Constructor of a non-owning 2D tensor. - * @param dimData initializer_list of DimData. - * @param dataPtr raw pointer to the source data array. - * @param isCPU whether to allocate tensor on CPU or GPU. - */ - Tensor2D(std::initializer_list dimData, DataType *dataPtr, bool isCPU) - : TensorBase(CT, dimData, dataPtr, isCPU) - { - } - - /** - * Get the length of the 2D tensor. - * @return length of the 2D tensor. - */ - std::size_t getLength() const - { - return getSize(DimToIndex2D::kLength); - } - - /** - * Get the channel count of the 2D tensor. - * @return channel count of the 2D tensor. - */ - std::size_t getChannelCount() const - { - return getSize(DimToIndex2D::kChannel); - } - - /** - * Expose base getStride() function. - */ - using TensorBase::getStride; - - /** - * Get the stride of the 2D tensor. - * @param dim tensor dimension. - * @return tensor stride of the given dimension. - */ - std::size_t getStride(TensorDimension dim) const - { - switch (dim) - { - case TensorDimension::LENGTH: - return getStride(DimToIndex2D::kLength); - case TensorDimension::CHANNEL: - return getStride(DimToIndex2D::kChannel); - default: - throw std::out_of_range("cvcore::Tensor2D::getStride ==> Requested TensorDimension is out of bounds"); - } - } - - /** - * Get the raw data pointer to the 2D tensor. - * @return data pointer to the 2D tensor. - */ - DataType *getData() - { - return reinterpret_cast(TensorBase::getData()); - } - - /** - * Get the const raw data pointer to the 2D tensor. - * @return const data pointer to the 2D tensor. - */ - const DataType *getData() const - { - return reinterpret_cast(TensorBase::getData()); - } -}; - -/** - * Implementation of 3D tensors. - * @tparam TL tensor layout type. - * @tparam CT channel type. - */ -template -class Tensor3D : public TensorBase -{ - using DataType = typename ChannelTypeToNative::Type; - -public: - /** - * Default Constructor. - */ - Tensor3D() = default; - - /** - * Constructor of a memory-owning 3D tensor. - * @param dimData initializer_list of DimData. - * @param isCPU whether to allocate tensor on CPU or GPU. - */ - Tensor3D(std::initializer_list dimData, bool isCPU) - : TensorBase(CT, dimData, isCPU) - { - } - - /** - * Constructor of a non-owning 3D tensor. - * @param dimData initializer_list of DimData. - * @param dataPtr raw pointer to the source data array. - * @param isCPU whether to allocate tensor on CPU or GPU. - */ - Tensor3D(std::initializer_list dimData, DataType *dataPtr, bool isCPU) - : TensorBase(CT, dimData, dataPtr, isCPU) - { - } - - /** - * Get the width of the 3D tensor. - * @return width of the 3D tensor. - */ - std::size_t getWidth() const - { - return getSize(DimToIndex3D::kWidth); - } - - /** - * Get the height of the 3D tensor. - * @return height of the 3D tensor. - */ - std::size_t getHeight() const - { - return getSize(DimToIndex3D::kHeight); - } - - /** - * Get the channel count of the 3D tensor. - * @return channel count of the 3D tensor. - */ - std::size_t getChannelCount() const - { - return getSize(DimToIndex3D::kChannel); - } - - /** - * Expose base getStride() function. - */ - using TensorBase::getStride; - - /** - * Get the stride of the 3D tensor. - * @param dim tensor dimension. - * @return tensor stride of the given dimension. - */ - std::size_t getStride(TensorDimension dim) const - { - switch (dim) - { - case TensorDimension::HEIGHT: - return getStride(DimToIndex3D::kHeight); - case TensorDimension::WIDTH: - return getStride(DimToIndex3D::kWidth); - case TensorDimension::CHANNEL: - return getStride(DimToIndex3D::kChannel); - default: - throw std::out_of_range("cvcore::Tensor3D::getStride ==> Requested TensorDimension is out of bounds"); - } - } - - /** - * Get the raw data pointer to the 3D tensor. - * @return data pointer to the 3D tensor. - */ - DataType *getData() - { - return reinterpret_cast(TensorBase::getData()); - } - - /** - * Get the const raw data pointer to the 3D tensor. - * @return const data pointer to the 3D tensor. - */ - const DataType *getData() const - { - return reinterpret_cast(TensorBase::getData()); - } -}; - -/** - * Implementation of 4D tensors. - * @tparam TL tensor layout type. - * @tparam CT channel type. - */ -template -class Tensor4D : public TensorBase -{ - using DataType = typename ChannelTypeToNative::Type; - -public: - /** - * Default Constructor. - */ - Tensor4D() = default; - - /** - * Constructor of a memory-owning 4D tensor. - * @param dimData initializer_list of DimData. - * @param isCPU whether to allocate tensor on CPU or GPU. - */ - Tensor4D(std::initializer_list dimData, bool isCPU) - : TensorBase(CT, dimData, isCPU) - { - } - - /** - * Constructor of a non-owning 4D tensor. - * @param dimData initializer_list of DimData. - * @param dataPtr raw pointer to the source data array. - * @param isCPU whether to allocate tensor on CPU or GPU. - */ - Tensor4D(std::initializer_list dimData, DataType *dataPtr, bool isCPU) - : TensorBase(CT, dimData, dataPtr, isCPU) - { - } - - /** - * Get the width of the 4D tensor. - * @return width of the 4D tensor. - */ - std::size_t getWidth() const - { - return getSize(DimToIndex4D::kWidth); - } - - /** - * Get the height of the 4D tensor. - * @return height of the 4D tensor. - */ - std::size_t getHeight() const - { - return getSize(DimToIndex4D::kHeight); - } - - /** - * Get the depth of the 4D tensor. - * @return depth of the 4D tensor. - */ - std::size_t getDepth() const - { - return getSize(DimToIndex4D::kDepth); - } - - /** - * Get the channel count of the 4D tensor. - * @return channel count of the 4D tensor. - */ - std::size_t getChannelCount() const - { - return getSize(DimToIndex4D::kChannel); - } - - /** - * Expose base getStride() function. - */ - using TensorBase::getStride; - - /** - * Get the stride of the 4D tensor. - * @param dim tensor dimension. - * @return tensor stride of the given dimension. - */ - std::size_t getStride(TensorDimension dim) const - { - switch (dim) - { - case TensorDimension::HEIGHT: - return getStride(DimToIndex4D::kHeight); - case TensorDimension::WIDTH: - return getStride(DimToIndex4D::kWidth); - case TensorDimension::CHANNEL: - return getStride(DimToIndex4D::kChannel); - case TensorDimension::DEPTH: - return getStride(DimToIndex4D::kDepth); - default: - throw std::out_of_range("cvcore::Tensor4D::getStride ==> Requested TensorDimension is out of bounds"); - } - } - - /** - * Get the raw data pointer to the 4D tensor. - * @return data pointer to the 4D tensor. - */ - DataType *getData() - { - return reinterpret_cast(TensorBase::getData()); - } - - /** - * Get the const raw data pointer to the 4D tensor. - * @return const data pointer to the 4D tensor. - */ - const DataType *getData() const - { - return reinterpret_cast(TensorBase::getData()); - } -}; - -} // namespace detail - -template -class Tensor; - -// 2D Tensors - -/** - * 2D LC tensors. - * @tparam CC channel count. - * @tparam CT channel type. - */ -template -class Tensor : public detail::Tensor2D -{ -public: - using DataType = typename detail::ChannelTypeToNative::Type; - - static constexpr ChannelCount kChannelCount = CC; - - Tensor() = default; - - template::type> - Tensor(std::size_t length, bool isCPU = true) - : detail::Tensor2D({{length, detail::ChannelToCount()}, {detail::ChannelToCount(), 1}}, isCPU) - { - } - - template::type> - Tensor(std::size_t length, std::size_t channelCount, bool isCPU = true) - : detail::Tensor2D({{length, channelCount}, {channelCount, 1}}, isCPU) - { - } - - template::type> - Tensor(std::size_t length, DataType *dataPtr, bool isCPU = true) - : detail::Tensor2D({{length, detail::ChannelToCount()}, {detail::ChannelToCount(), 1}}, dataPtr, - isCPU) - { - } - - template::type> - Tensor(std::size_t length, std::size_t channelCount, DataType *dataPtr, bool isCPU = true) - : detail::Tensor2D({{length, channelCount}, {channelCount, 1}}, dataPtr, isCPU) - { - } -}; - -/** - * 2D CL tensors. - * @tparam CC channel count. - * @tparam CT channel type. - */ -template -class Tensor : public detail::Tensor2D -{ -public: - using DataType = typename detail::ChannelTypeToNative::Type; - - static constexpr ChannelCount kChannelCount = CC; - - Tensor() = default; - - template::type> - Tensor(std::size_t length, bool isCPU = true) - : detail::Tensor2D({{detail::ChannelToCount(), length}, {length, 1}}, isCPU) - { - } - - template::type> - Tensor(std::size_t length, std::size_t channelCount, bool isCPU = true) - : detail::Tensor2D({{channelCount, length}, {length, 1}}, isCPU) - { - } - - template::type> - Tensor(std::size_t length, DataType *dataPtr, bool isCPU = true) - : detail::Tensor2D({{detail::ChannelToCount(), length}, {length, 1}}, dataPtr, isCPU) - { - } - - template::type> - Tensor(std::size_t length, std::size_t channelCount, DataType *dataPtr, bool isCPU = true) - : detail::Tensor2D({{channelCount, length}, {length, 1}}, dataPtr, isCPU) - { - } -}; - -// 3D Tensors - -/** - * 3D HWC tensors. - * @tparam CC channel count. - * @tparam CT channel type. - */ -template -class Tensor : public detail::Tensor3D -{ -public: - using DataType = typename detail::ChannelTypeToNative::Type; - - static constexpr ChannelCount kChannelCount = CC; - - Tensor() = default; - - template::value>::type * = nullptr> - Tensor(std::size_t width, std::size_t height, B isCPU = true) - : detail::Tensor3D({{height, width * detail::ChannelToCount()}, - {width, detail::ChannelToCount()}, - {detail::ChannelToCount(), 1}}, - isCPU) - { - } - - template::value>::type * = nullptr> - Tensor(std::size_t width, std::size_t height, std::size_t channelCount, B isCPU = true) - : detail::Tensor3D({{height, width * channelCount}, {width, channelCount}, {channelCount, 1}}, isCPU) - { - } - - template::value>::type * = nullptr> - Tensor(std::size_t width, std::size_t height, DataType *dataPtr, B isCPU = true) - : detail::Tensor3D({{height, width * detail::ChannelToCount()}, - {width, detail::ChannelToCount()}, - {detail::ChannelToCount(), 1}}, - dataPtr, isCPU) - { - } - - template::value>::type * = nullptr> - Tensor(std::size_t width, std::size_t height, std::size_t rowPitch, DataType *dataPtr, B isCPU = true) - : detail::Tensor3D({{height, rowPitch / GetChannelSize(CT)}, - {width, detail::ChannelToCount()}, - {detail::ChannelToCount(), 1}}, - dataPtr, isCPU) - { - if (rowPitch % GetChannelSize(CT) != 0) - { - throw std::domain_error( - "cvcore::Tensor::Tensor ==> Parameter rowPitch is not evenly divisible by channel size"); - } - } - - template::value>::type * = nullptr> - Tensor(std::size_t width, std::size_t height, std::size_t channelCount, DataType *dataPtr, B isCPU = true) - : detail::Tensor3D({{height, width * channelCount}, {width, channelCount}, {channelCount, 1}}, dataPtr, - isCPU) - { - } - - template::value>::type * = nullptr> - Tensor(std::size_t width, std::size_t height, std::size_t channelCount, std::size_t rowPitch, DataType *dataPtr, - B isCPU = true) - : detail::Tensor3D({{height, rowPitch / GetChannelSize(CT)}, {width, channelCount}, {channelCount, 1}}, - dataPtr, isCPU) - { - if (rowPitch % GetChannelSize(CT) != 0) - { - throw std::domain_error( - "cvcore::Tensor::Tensor ==> Parameter rowPitch is not evenly divisible by channel size"); - } - } -}; - -/** - * 3D CHW tensors. - * @tparam CC channel count. - * @tparam CT channel type. - */ -template -class Tensor : public detail::Tensor3D -{ -public: - using DataType = typename detail::ChannelTypeToNative::Type; - - static constexpr ChannelCount kChannelCount = CC; - - Tensor() = default; - - template::type> - Tensor(std::size_t width, std::size_t height, bool isCPU = true) - : detail::Tensor3D({{detail::ChannelToCount(), width * height}, {height, width}, {width, 1}}, - isCPU) - { - } - - template::type> - Tensor(std::size_t width, std::size_t height, std::size_t channelCount, bool isCPU = true) - : detail::Tensor3D({{channelCount, width * height}, {height, width}, {width, 1}}, isCPU) - { - } - - template::type> - Tensor(std::size_t width, std::size_t height, DataType *dataPtr, bool isCPU = true) - : detail::Tensor3D({{detail::ChannelToCount(), width * height}, {height, width}, {width, 1}}, - dataPtr, isCPU) - { - } - - template::value>::type * = nullptr> - Tensor(std::size_t width, std::size_t height, std::size_t rowPitch, DataType *dataPtr, B isCPU = true) - : detail::Tensor3D({{detail::ChannelToCount(), height * rowPitch / GetChannelSize(CT)}, - {height, rowPitch / GetChannelSize(CT)}, - {width, 1}}, - dataPtr, isCPU) - { - if (rowPitch % GetChannelSize(CT) != 0) - { - throw std::domain_error( - "cvcore::Tensor::Tensor ==> Parameter rowPitch is not evenly divisible by channel size"); - } - } - - template::type> - Tensor(std::size_t width, std::size_t height, std::size_t channelCount, DataType *dataPtr, bool isCPU = true) - : detail::Tensor3D({{channelCount, width * height}, {height, width}, {width, 1}}, dataPtr, isCPU) - { - } - - template::value>::type * = nullptr> - Tensor(std::size_t width, std::size_t height, std::size_t channelCount, std::size_t rowPitch, DataType *dataPtr, - B isCPU = true) - : detail::Tensor3D({{channelCount, height * rowPitch / GetChannelSize(CT)}, - {height, rowPitch / GetChannelSize(CT)}, - {width, 1}}, - dataPtr, isCPU) - { - if (rowPitch % GetChannelSize(CT) != 0) - { - throw std::domain_error( - "cvcore::Tensor::Tensor ==> Parameter rowPitch is not evenly divisible by channel size"); - } - } -}; - -// 4D Tensors - -/** - * 4D DHWC tensors. - * @tparam CC channel count. - * @tparam CT channel type. - */ -template -class Tensor : public detail::Tensor4D -{ -public: - using DataType = typename detail::ChannelTypeToNative::Type; - - static constexpr ChannelCount kChannelCount = CC; - - Tensor() = default; - - template::value>::type * = nullptr> - Tensor(std::size_t width, std::size_t height, std::size_t depth, B isCPU = true) - : detail::Tensor4D({{depth, height * width * detail::ChannelToCount()}, - {height, width * detail::ChannelToCount()}, - {width, detail::ChannelToCount()}, - {detail::ChannelToCount(), 1}}, - isCPU) - { - } - - template::value>::type * = nullptr> - Tensor(std::size_t width, std::size_t height, std::size_t depth, std::size_t channelCount, B isCPU = true) - : detail::Tensor4D({{depth, height * width * channelCount}, - {height, width * channelCount}, - {width, channelCount}, - {channelCount, 1}}, - isCPU) - { - } - - template::value>::type * = nullptr> - Tensor(std::size_t width, std::size_t height, std::size_t depth, DataType *dataPtr, B isCPU = true) - : detail::Tensor4D({{depth, height * width * detail::ChannelToCount()}, - {height, width * detail::ChannelToCount()}, - {width, detail::ChannelToCount()}, - {detail::ChannelToCount(), 1}}, - dataPtr, isCPU) - { - } - - template::value>::type * = nullptr> - Tensor(std::size_t width, std::size_t height, std::size_t depth, std::size_t rowPitch, DataType *dataPtr, - B isCPU = true) - : detail::Tensor4D({{depth, height * rowPitch / GetChannelSize(CT)}, - {height, rowPitch / GetChannelSize(CT)}, - {width, detail::ChannelToCount()}, - {detail::ChannelToCount(), 1}}, - dataPtr, isCPU) - { - if (rowPitch % GetChannelSize(CT) != 0) - { - throw std::domain_error( - "cvcore::Tensor::Tensor ==> Parameter rowPitch is not evenly divisible by channel size"); - } - } - - template::value>::type * = nullptr> - Tensor(std::size_t width, std::size_t height, std::size_t depth, std::size_t channelCount, DataType *dataPtr, - B isCPU = true) - : detail::Tensor4D({{depth, height * width * channelCount}, - {height, width * channelCount}, - {width, channelCount}, - {channelCount, 1}}, - dataPtr, isCPU) - { - } - - template::value>::type * = nullptr> - Tensor(std::size_t width, std::size_t height, std::size_t depth, std::size_t channelCount, std::size_t rowPitch, - DataType *dataPtr, B isCPU = true) - : detail::Tensor4D({{depth, height * rowPitch / GetChannelSize(CT)}, - {height, rowPitch / GetChannelSize(CT)}, - {width, channelCount}, - {channelCount, 1}}, - dataPtr, isCPU) - { - if (rowPitch % GetChannelSize(CT) != 0) - { - throw std::domain_error( - "cvcore::Tensor::Tensor ==> Parameter rowPitch is not evenly divisible by channel size"); - } - } -}; - -/** - * 4D DCHW tensors. - * @tparam CC channel count. - * @tparam CT channel type. - */ -template -class Tensor : public detail::Tensor4D -{ -public: - using DataType = typename detail::ChannelTypeToNative::Type; - - static constexpr ChannelCount kChannelCount = CC; - - Tensor() = default; - - template::type> - Tensor(std::size_t width, std::size_t height, std::size_t depth, bool isCPU = true) - : detail::Tensor4D({{depth, detail::ChannelToCount() * width * height}, - {detail::ChannelToCount(), width * height}, - {height, width}, - {width, 1}}, - isCPU) - { - } - - template::type> - Tensor(std::size_t width, std::size_t height, std::size_t depth, std::size_t channelCount, bool isCPU = true) - : detail::Tensor4D( - {{depth, channelCount * width * height}, {channelCount, width * height}, {height, width}, {width, 1}}, - isCPU) - { - } - - template::type> - Tensor(std::size_t width, std::size_t height, std::size_t depth, DataType *dataPtr, bool isCPU = true) - : detail::Tensor4D({{depth, detail::ChannelToCount() * width * height}, - {detail::ChannelToCount(), width * height}, - {height, width}, - {width, 1}}, - dataPtr, isCPU) - { - } - - template::value>::type * = nullptr> - Tensor(std::size_t width, std::size_t height, std::size_t depth, std::size_t rowPitch, DataType *dataPtr, - B isCPU = true) - : detail::Tensor4D({{depth, detail::ChannelToCount() * height * rowPitch / GetChannelSize(CT)}, - {detail::ChannelToCount(), height * rowPitch / GetChannelSize(CT)}, - {height, rowPitch / GetChannelSize(CT)}, - {width, 1}}, - dataPtr, isCPU) - { - if (rowPitch % GetChannelSize(CT) != 0) - { - throw std::domain_error( - "cvcore::Tensor::Tensor ==> Parameter rowPitch is not evenly divisible by channel size"); - } - } - - template::type> - Tensor(std::size_t width, std::size_t height, std::size_t depth, std::size_t channelCount, DataType *dataPtr, - bool isCPU = true) - : detail::Tensor4D( - {{depth, channelCount * width * height}, {channelCount, width * height}, {height, width}, {width, 1}}, - dataPtr, isCPU) - { - } - - template::value>::type * = nullptr> - Tensor(std::size_t width, std::size_t height, std::size_t depth, std::size_t channelCount, std::size_t rowPitch, - DataType *dataPtr, B isCPU = true) - : detail::Tensor4D({{depth, channelCount * height * rowPitch / GetChannelSize(CT)}, - {channelCount, height * rowPitch / GetChannelSize(CT)}, - {height, rowPitch / GetChannelSize(CT)}, - {width, 1}}, - dataPtr, isCPU) - { - if (rowPitch % GetChannelSize(CT) != 0) - { - throw std::domain_error( - "cvcore::Tensor::Tensor ==> Parameter rowPitch is not evenly divisible by channel size"); - } - } -}; - -/** - * 4D CDHW tensors. - * @tparam CC channel count. - * @tparam CT channel type. - */ -template -class Tensor : public detail::Tensor4D -{ -public: - using DataType = typename detail::ChannelTypeToNative::Type; - - static constexpr ChannelCount kChannelCount = CC; - - Tensor() = default; - - template::type> - Tensor(std::size_t width, std::size_t height, std::size_t depth, bool isCPU = true) - : detail::Tensor4D({{detail::ChannelToCount(), depth * width * height}, - {depth, width * height}, - {height, width}, - {width, 1}}, - isCPU) - { - } - - template::type> - Tensor(std::size_t width, std::size_t height, std::size_t depth, std::size_t channelCount, bool isCPU = true) - : detail::Tensor4D( - {{channelCount, depth * width * height}, {depth, width * height}, {height, width}, {width, 1}}, isCPU) - { - } - - template::type> - Tensor(std::size_t width, std::size_t height, std::size_t depth, DataType *dataPtr, bool isCPU = true) - : detail::Tensor4D({{detail::ChannelToCount(), depth * width * height}, - {depth, width * height}, - {height, width}, - {width, 1}}, - dataPtr, isCPU) - { - } - - template::type> - Tensor(std::size_t width, std::size_t height, std::size_t depth, std::size_t channelCount, DataType *dataPtr, - bool isCPU = true) - : detail::Tensor4D( - {{channelCount, depth * width * height}, {depth, width * height}, {height, width}, {width, 1}}, dataPtr, - isCPU) - { - } -}; - -} // namespace cvcore - -#endif // CVCORE_TENSOR_H diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/core/TensorList.h b/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/core/TensorList.h deleted file mode 100644 index 4ec6695..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/core/TensorList.h +++ /dev/null @@ -1,37 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2019-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -#ifndef CVCORE_TENSORLIST_H -#define CVCORE_TENSORLIST_H - -#include - -#include "Traits.h" -#include "Array.h" -#include "Tensor.h" - -namespace cvcore { - -/** - * @brief Implementation of a list of tensors of the same rank but, potentially, different dimensions - */ -template -using TensorList = typename std::enable_if::value, Array>::type; - -} // namespace cvcore - -#endif // CVCORE_TENSORLIST_H diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/core/TensorMap.h b/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/core/TensorMap.h deleted file mode 100644 index 93c8783..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/core/TensorMap.h +++ /dev/null @@ -1,534 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2019-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -#ifndef CVCORE_TENSORMAP_H -#define CVCORE_TENSORMAP_H - -#include -#include -#include -#include -#include - -#include "Traits.h" -#include "TensorList.h" - -namespace cvcore { - -/** - * @brief Implementation of a map of tensors of the same rank but, potentially, different dimensions, over the batch dimension - * - * @tparam TensorType Any CVCORE tensor type - * @tparam KeyType Any STL hashable data type - */ -template -class TensorMap {}; - -template -class TensorMap, KT, - typename std::enable_if>::value>::type> -{ - using my_type = TensorMap, KT>; - - public: - using key_type = KT; - using unit_type = Tensor; - using element_type = traits::remove_batch_t; - using frame_type = TensorList; - using buffer_type = TensorList; - - template - struct dim_data_type - { - std::size_t height; - std::size_t width; - }; - - template - struct dim_data_type::type> - { - std::size_t height; - std::size_t width; - std::size_t channels; - }; - - TensorMap() = default; - TensorMap(const my_type &) = delete; - TensorMap(my_type && other) - { - *this = std::move(other); - } - - /** - * @brief Construct a new Tensor Map object - * - * @param batchSize The batch dimension of all sub-tensors - * @param dimData The dimensional description of all sub-tensors in at least HWC format - * @param isCPU A boolean flag specifying what device to allocate the sub-tensors - */ - template::type> - TensorMap(std::size_t batchSize, - const std::vector> & dimData, - bool isCPU = true) - : m_maxBatchSize{batchSize}, m_size{dimData.size()}, - m_isCPU{isCPU}, m_buffer{dimData.size(), true} - { - m_buffer.setSize(m_size); - - int i = 0; - for(auto dim: dimData) - { - m_buffer[i] = std::move(unit_type(dim.width, - dim.height, - m_maxBatchSize, - m_isCPU)); - ++i; - } - - for(std::size_t i = 0; i < m_maxBatchSize; ++i) - { - m_pool.insert(i); - } - } - - /** - * @brief Construct a new Tensor Map object - * - * @param batchSize The batch dimension of all sub-tensors - * @param dimData The dimensional description of all sub-tensors in at least HWC format - * @param isCPU A boolean flag specifying what device to allocate the sub-tensors - */ - template::type> - TensorMap(std::size_t batchSize, - const std::vector> & dimData, - bool isCPU = true) - : m_maxBatchSize{batchSize}, m_size{dimData.size()}, - m_isCPU{isCPU}, m_buffer{dimData.size(), true} - { - m_buffer.setSize(m_size); - - int i = 0; - for(auto dim: dimData) - { - m_buffer[i] = std::move(unit_type(dim.width, - dim.height, - m_maxBatchSize, - dim.channels, - m_isCPU)); - ++i; - } - - for(std::size_t i = 0; i < m_maxBatchSize; ++i) - { - m_pool.insert(i); - } - } - - ~TensorMap() = default; - - my_type & operator=(const my_type &) = delete; - my_type & operator=(my_type && other) - { - std::swap(m_mapping, other.m_mapping); - std::swap(m_pool, other.m_pool); - std::swap(m_maxBatchSize, other.m_maxBatchSize); - std::swap(m_size, other.m_size); - std::swap(m_isCPU, other.m_isCPU); - std::swap(m_buffer, other.m_buffer); - - return *this; - } - - /** - * @brief A mapping of the batch dimension index to a given key - * - * @details Given a set of pairs such that the keys AND values are unique - * respectively, the key-wise mapping of the batch dimension is reset - * to the provided values. - * - * @param pairs An unordered map of the uniqe key value pairs - * @return true If the length of ``pairs`` is less than the max batch size - * and the key value pairs are one-to-one and onto. - * @return false If the conditions of ``true`` are not met. - */ - bool remap(const std::unordered_map & pairs) - { - bool result = false; - - if(pairs.size() <= m_maxBatchSize) - { - for(std::size_t i = 0; i < m_maxBatchSize; ++i) - { - m_pool.insert(i); - } - - m_mapping.clear(); - for(auto mapping: pairs) - { - if(m_pool.erase(mapping.second)) - { - m_mapping[mapping.first] = mapping.second; - } - } - - if((pairs.size() + m_pool.size()) == m_maxBatchSize) - { - result = true; - } - } - - return result; - } - - /** - * @brief Associates a given key with the first available batch index - * - * @details Assuming the associated keys has not reached `maxBatchSize`` - * then this function associates a given key with the first available - * batch index and returns that index value. If no batch index is - * available -1 is returned. NOTE: if ``key`` is already associated - * with a batch index, the that index is returned. - * - * @param key The key to be associated with a batch index value - * @return std::intmax_t The batch index associated with the key or -1 - * if no index is available. NOTE: because std::intmax_t is not a full - * covering of std::size_t, it is possible for wrap around to happen. - */ - std::intmax_t map(const key_type & key) - { - auto it = m_mapping.find(key); - - if(it == m_mapping.end() && !m_pool.empty()) - { - auto value = m_pool.begin(); - it = m_mapping.insert({key, *value}).first; - m_pool.erase(value); - } - - return static_cast(it != m_mapping.end() ? it->second : -1); - } - - /** - * @brief Dissociates a given key with a batch index if possible - * - * @details Assuming the given key is associated with a batch index this - * function removes the association and returns the batch index is was - * associated with. If no batch index is found associated with the given - * key, -1 is returned. - * - * @param key The key to be dissociated - * @return std::intmax_t The batch index associated with the key or -1 - * if not found. NOTE: because std::intmax_t is not a full covering of - * std::size_t, it is possible for wrap around to happen. - */ - std::intmax_t unmap(const key_type & key) - { - std::intmax_t result = -1; - - auto it = m_mapping.find(key); - - if(it != m_mapping.end()) - { - result = static_cast(it->second); - m_pool.insert(it->second); - m_mapping.erase(it); - } - - return result; - } - - /** - * @brief The number of keys associated with a batch index - * - * @return std::size_t - */ - std::size_t getKeyCount() const noexcept - { - return m_mapping.size(); - } - - /** - * @brief The maximum number of batch index - * - * @return std::size_t - */ - std::size_t getMaxBatchSize() const noexcept - { - return m_maxBatchSize; - } - - /** - * @brief The number of sub-tensors - * - * @return std::size_t - */ - std::size_t getUnitCount() const - { - return m_size; - } - - /** - * Get the size of given dimension. - * @param dimIdx dimension index. - * @return size of the specified dimension. - */ - std::size_t getTensorSize(std::size_t unitIdx, std::size_t dimIdx) const - { - return m_buffer[unitIdx].getSize(dimIdx); - } - - /** - * Get the stride of given dimension. - * @param dimIdx dimension index. - * @return stride of the specified dimension. - */ - std::size_t getTensorStride(std::size_t unitIdx, std::size_t dimIdx) const - { - return m_buffer[unitIdx].getStride(dimIdx); - } - - template::type> - unit_type getUnit(std::size_t idx) - { - unit_type result{m_buffer[idx].getWidth(), - m_buffer[idx].getHeight(), - m_buffer[idx].getDepth(), - m_buffer[idx].getData(), - m_buffer[idx].isCPU()}; - return result; - } - - template::type> - unit_type getUnit(std::size_t idx, ChannelCount UNUSED = T) - { - unit_type result{m_buffer[idx].getWidth(), - m_buffer[idx].getHeight(), - m_buffer[idx].getDepth(), - m_buffer[idx].getChannelCount(), - m_buffer[idx].getData(), - m_buffer[idx].isCPU()}; - return result; - } - - template::type> - frame_type getFrame(const key_type & idx) - { - frame_type result; - - if(m_mapping.find(idx) != m_mapping.end()) - { - std::size_t at = m_mapping[idx]; - result = std::move(frame_type{m_buffer.getSize(), m_buffer.isCPU()}); - result.setSize(m_size); - for(std::size_t i = 0; i < m_size; ++i) - { - element_type element{m_buffer[i].getWidth(), - m_buffer[i].getHeight(), - m_buffer[i].getData() + - at * m_buffer[i].getStride(TensorDimension::DEPTH), - m_buffer[i].isCPU()}; - result[i] = std::move(element); - } - } - - return result; - } - - template::type> - frame_type getFrame(const key_type & idx, ChannelCount UNUSED = T) - { - frame_type result; - - if(m_mapping.find(idx) != m_mapping.end()) - { - std::size_t at = m_mapping[idx]; - result = std::move(frame_type{m_buffer.getSize(), m_buffer.isCPU()}); - result.setSize(m_size); - for(std::size_t i = 0; i < m_size; ++i) - { - element_type element{m_buffer[i].getWidth(), - m_buffer[i].getHeight(), - m_buffer[i].getChannelCount(), - m_buffer[i].getData() + - at * m_buffer[i].getStride(TensorDimension::DEPTH), - m_buffer[i].isCPU()}; - result[i] = std::move(element); - } - } - - return result; - } - - template::type> - frame_type getFrame(key_type && idx) - { - frame_type result; - - if(m_mapping.find(idx) != m_mapping.end()) - { - std::size_t at = m_mapping[idx]; - result = std::move(frame_type{m_buffer.getSize(), m_buffer.isCPU()}); - result.setSize(m_size); - for(std::size_t i = 0; i < m_size; ++i) - { - element_type element{m_buffer[i].getWidth(), - m_buffer[i].getHeight(), - m_buffer[i].getData() + - at * m_buffer[i].getStride(TensorDimension::DEPTH), - m_buffer[i].isCPU()}; - result[i] = std::move(element); - } - } - - return result; - } - - template::type> - frame_type getFrame(key_type && idx, ChannelCount UNUSED = T) - { - frame_type result; - - if(m_mapping.find(idx) != m_mapping.end()) - { - std::size_t at = m_mapping[idx]; - result = std::move(frame_type{m_buffer.getSize(), m_buffer.isCPU()}); - result.setSize(m_size); - for(std::size_t i = 0; i < m_size; ++i) - { - element_type element{m_buffer[i].getWidth(), - m_buffer[i].getHeight(), - m_buffer[i].getChannelCount(), - m_buffer[i].getData() + - at * m_buffer[i].getStride(TensorDimension::DEPTH), - m_buffer[i].isCPU()}; - result[i] = std::move(element); - } - } - - return result; - } - - template::type> - element_type getElement(const key_type & keyIdx, std::size_t unitIdx) - { - element_type element; - - if(m_mapping.find(keyIdx) != m_mapping.end()) - { - std::size_t at = m_mapping[keyIdx]; - element = std::move(element_type{m_buffer[unitIdx].getWidth(), - m_buffer[unitIdx].getHeight(), - m_buffer[unitIdx].getData() + - at * m_buffer[unitIdx].getStride(TensorDimension::DEPTH), - m_buffer[unitIdx].isCPU()}); - } - - return element; - } - - template::type> - element_type getElement(const key_type & keyIdx, std::size_t unitIdx, ChannelCount UNUSED = T) - { - element_type element; - - if(m_mapping.find(keyIdx) != m_mapping.end()) - { - std::size_t at = m_mapping[keyIdx]; - element = std::move(element_type{m_buffer[unitIdx].getWidth(), - m_buffer[unitIdx].getHeight(), - m_buffer[unitIdx].getChannelCount(), - m_buffer[unitIdx].getData() + - at * m_buffer[unitIdx].getStride(TensorDimension::DEPTH), - m_buffer[unitIdx].isCPU()}); - } - - return element; - } - - template::type> - element_type getElement(key_type && keyIdx, std::size_t unitIdx) - { - element_type element; - - if(m_mapping.find(keyIdx) != m_mapping.end()) - { - std::size_t at = m_mapping[keyIdx]; - element = std::move(element_type{m_buffer[unitIdx].getWidth(), - m_buffer[unitIdx].getHeight(), - m_buffer[unitIdx].getData() + - at * m_buffer[unitIdx].getStride(TensorDimension::DEPTH), - m_buffer[unitIdx].isCPU()}); - } - - return element; - } - - template::type> - element_type getElement(key_type && keyIdx, std::size_t unitIdx, ChannelCount UNUSED = T) - { - element_type element; - - if(m_mapping.find(keyIdx) != m_mapping.end()) - { - std::size_t at = m_mapping[keyIdx]; - element = std::move(element_type{m_buffer[unitIdx].getWidth(), - m_buffer[unitIdx].getHeight(), - m_buffer[unitIdx].getChannelCount(), - m_buffer[unitIdx].getData() + - at * m_buffer[unitIdx].getStride(TensorDimension::DEPTH), - m_buffer[unitIdx].isCPU()}); - } - - return element; - } - - /** - * Get the ChannelType of the Tensor. - * @return ChannelType of the Tensor. - */ - constexpr ChannelType getType() const noexcept - { - return CT; - } - - /** - * Get the flag whether the Tensor is allocated in CPU or GPU. - * @return whether the Tensor is allocated in CPU. - */ - bool isCPU() const noexcept - { - return m_isCPU; - } - - private: - // Mapping and Pool form a unique-to-unique isometry between - // the keys and indices of the batch dimension - mutable std::unordered_map m_mapping; - mutable std::set m_pool; - - std::size_t m_maxBatchSize; - std::size_t m_size; - bool m_isCPU; - - buffer_type m_buffer; -}; - -} // namespace cvcore - -#endif // CVCORE_TENSORMAP_H diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/core/Traits.h b/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/core/Traits.h deleted file mode 100644 index a78b780..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/core/Traits.h +++ /dev/null @@ -1,478 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -#ifndef CVCORE_TRAITS_H -#define CVCORE_TRAITS_H - -#include - -#include "MathTypes.h" -#include "Tensor.h" - -namespace cvcore { namespace traits { - -// ----------------------------------------------------------------------------- -// Type Properties -// ----------------------------------------------------------------------------- - -template -struct is_tensor : std::false_type -{ -}; - -template -struct is_tensor> : std::true_type -{ -}; - -template -struct is_planar : std::false_type -{ - static_assert(is_tensor::value, ""); -}; - -template -struct is_planar> : std::integral_constant -{ -}; - -template -struct is_interleaved : std::false_type -{ - static_assert(is_tensor::value, ""); -}; - -template -struct is_interleaved> : std::integral_constant -{ -}; - -template -struct is_batch : std::false_type -{ - static_assert(is_tensor::value, ""); -}; - -template -struct is_batch> : std::integral_constant -{ -}; - -// ----------------------------------------------------------------------------- -// Type Modifications -// ----------------------------------------------------------------------------- - -template -struct to_planar -{ - static_assert(is_tensor::value, ""); - - using type = TensorType; -}; - -template -struct to_planar> -{ - using type = Tensor; -}; - -template -struct to_planar> -{ - using type = Tensor; -}; - -template -using to_planar_t = typename to_planar::type; - -template -struct to_interleaved -{ - static_assert(is_tensor::value, ""); - - using type = TensorType; -}; - -template -struct to_interleaved> -{ - using type = Tensor; -}; - -template -struct to_interleaved> -{ - using type = Tensor; -}; - -template -struct to_interleaved> -{ - using type = Tensor; -}; - -template -using to_interleaved_t = typename to_interleaved::type; - -template -struct add_batch -{ - static_assert(is_tensor::value, ""); - - using type = TensorType; -}; - -template -struct add_batch> -{ - using type = Tensor; -}; - -template -struct add_batch> -{ - using type = Tensor; -}; - -template -using add_batch_t = typename add_batch::type; - -template -struct remove_batch -{ - static_assert(is_tensor::value, ""); - - using type = TensorType; -}; - -template -struct remove_batch> -{ - using type = Tensor; -}; - -template -struct remove_batch> -{ - using type = Tensor; -}; - -template -struct remove_batch> -{ - using type = Tensor; -}; - -template -using remove_batch_t = typename remove_batch::type; - -template -struct to_c1 -{ - static_assert(is_tensor::value, ""); -}; - -template -struct to_c1> -{ - using type = Tensor; -}; - -template -using to_c1_t = typename to_c1::type; - -template -struct to_c2 -{ - static_assert(is_tensor::value, ""); -}; - -template -struct to_c2> -{ - using type = Tensor; -}; - -template -using to_c2_t = typename to_c2::type; - -template -struct to_c3 -{ - static_assert(is_tensor::value, ""); -}; - -template -struct to_c3> -{ - using type = Tensor; -}; - -template -using to_c3_t = typename to_c3::type; - -template -struct to_c4 -{ - static_assert(is_tensor::value, ""); -}; - -template -struct to_c4> -{ - using type = Tensor; -}; - -template -using to_c4_t = typename to_c4::type; - -template -struct to_cx -{ - static_assert(is_tensor::value, ""); -}; - -template -struct to_cx> -{ - using type = Tensor; -}; - -template -using to_cx_t = typename to_cx::type; - -template -struct to_u8 -{ - static_assert(is_tensor::value, ""); -}; - -template -struct to_u8> -{ - using type = Tensor; -}; - -template -using to_u8_t = typename to_u8::type; - -template -struct to_u16 -{ - static_assert(is_tensor::value, ""); -}; - -template -struct to_u16> -{ - using type = Tensor; -}; - -template -using to_u16_t = typename to_u16::type; - -template -struct to_f16 -{ - static_assert(is_tensor::value, ""); -}; - -template -struct to_f16> -{ - using type = Tensor; -}; - -template -using to_f16_t = typename to_f16::type; - -template -struct to_f32 -{ - static_assert(is_tensor::value, ""); -}; - -template -struct to_f32> -{ - using type = Tensor; -}; - -template -using to_f32_t = typename to_f32::type; - -template -using to_c1u8 = to_c1>; - -template -using to_c1u16 = to_c1>; - -template -using to_c1f16 = to_c1>; - -template -using to_c1f32 = to_c1>; - -template -using to_c2u8 = to_c2>; - -template -using to_c2u16 = to_c2>; - -template -using to_c2f16 = to_c2>; - -template -using to_c2f32 = to_c2>; - -template -using to_c3u8 = to_c3>; - -template -using to_c3u16 = to_c3>; - -template -using to_c3f16 = to_c3>; - -template -using to_c3f32 = to_c3>; - -template -using to_c4u8 = to_c4>; - -template -using to_c4u16 = to_c4>; - -template -using to_c4f16 = to_c4>; - -template -using to_c4f32 = to_c4>; - -template -using to_cxu8 = to_cx>; - -template -using to_cxu16 = to_cx>; - -template -using to_cxf16 = to_cx>; - -template -using to_cxf32 = to_cx>; - -template -using to_c1u8_t = to_c1_t>; - -template -using to_c1u16_t = to_c1_t>; - -template -using to_c1f16_t = to_c1_t>; - -template -using to_c1f32_t = to_c1_t>; - -template -using to_c2u8_t = to_c2_t>; - -template -using to_c2u16_t = to_c2_t>; - -template -using to_c2f16_t = to_c2_t>; - -template -using to_c2f32_t = to_c2_t>; - -template -using to_c3u8_t = to_c3_t>; - -template -using to_c3u16_t = to_c3_t>; - -template -using to_c3f16_t = to_c3_t>; - -template -using to_c3f32_t = to_c3_t>; - -template -using to_c4u8_t = to_c4_t>; - -template -using to_c4u16_t = to_c4_t>; - -template -using to_c4f16_t = to_c4_t>; - -template -using to_c4f32_t = to_c4_t>; - -template -using to_cxu8_t = to_cx_t>; - -template -using to_cxu16_t = to_cx_t>; - -template -using to_cxf16_t = to_cx_t>; - -template -using to_cxf32_t = to_cx_t>; - -template -struct get_dim; - -template<> -struct get_dim -{ - static constexpr int value = 1; -}; - -template<> -struct get_dim -{ - static constexpr int value = 1; -}; - -template<> -struct get_dim -{ - static constexpr int value = 2; -}; - -template<> -struct get_dim -{ - static constexpr int value = 2; -}; - -template<> -struct get_dim -{ - static constexpr int value = 3; -}; - -template<> -struct get_dim -{ - static constexpr int value = 3; -}; -}} // namespace cvcore::traits - -#endif // CVCORE_TRAITS_H diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/tensor_ops/BBoxUtils.h b/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/tensor_ops/BBoxUtils.h deleted file mode 100644 index 624139b..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/tensor_ops/BBoxUtils.h +++ /dev/null @@ -1,135 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2020-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -#ifndef CVCORE_BBOX_UTILS_H -#define CVCORE_BBOX_UTILS_H - -#include "cv/core/BBox.h" - -namespace cvcore { namespace tensor_ops { - -/** - * An enum. - * Enum type for Bounding Box interpolation - */ -enum BBoxInterpolationType -{ - CONST_INTERPOLATION, /**< Uses constant value interpolation */ - IMAGE_INTERPOLATION, /**< Interpolates based on image size */ -}; - -/** - * An enum. - * Enum type for Bounding Box scaling operation. - */ -enum BBoxScaleType -{ - NORMAL, /**< Scale box without fixing center point. */ - CENTER /**< Scale box with center fixed. */ -}; - -/** - * Function to calculate the intersection of two bounding boxes. - * @param a one of the BBox. - * @param b the other BBox. - * @return intersection area of the two bounding boxes. - */ -float GetIntersection(const BBox &a, const BBox &b); - -/** - * Function to calculate the union of two bounding boxes. - * @param a one of the BBox. - * @param b the other BBox. - * @return union area of the two bounding boxes. - */ -float GetUnion(const BBox &a, const BBox &b); - -/** - * Function to calculate the IoU (Intersection over Union) of two bounding boxes. - * @param a one of the BBox. - * @param b the other BBox. - * @return IoU of the two bounding boxes. - */ -float GetIoU(const BBox &a, const BBox &b); - -/** - * Function to merge two BBox together. - * @param a one of the BBox. - * @param b the other BBox. - * @return Merged bounding box. - */ -BBox MergeBoxes(const BBox &a, const BBox &b); - -/** - * Clamp BBox. - * @param a BBox to be clamped. - * @param b boundary BBox. - * @return clamped BBox. - */ -BBox ClampBox(const BBox &a, const BBox &b); - -/** - * Interpolate bounding boxes. - * @param currLeft left x coordinate. - * @param currRight right x coordinate. - * @param currBottom bottom y coordinate. - * @param currTop top y coordiante. - * @param xScaler scale ratio along width direction. - * @param yScaler scale ratio along height direction. - * @param currColumn current column index. - * @param currRow current row index. - * @param type bbox interpolation type. - * @param bboxNorm bounding box scaled factor. - * @return interpolated BBox. - */ -BBox InterpolateBoxes(float currLeft, float currRight, float currBottom, float currTop, float xScaler, float yScaler, - int currColumn, int currRow, BBoxInterpolationType type = IMAGE_INTERPOLATION, - float bboxNorm = 1.0); - -/** - * Scale BBox. - * @param bbox input BBox. - * @param xScaler scale ratio along width direction. - * @param yScaler scale ratio along height direction. - * @param type BBox scaling type. - * @return scaled BBox. - */ -BBox ScaleBox(const BBox &bbox, float xScaler, float yScaler, BBoxScaleType type = NORMAL); - -/** - * Transform BBox. - * @param bbox input BBox. - * @param xScaler scale ratio along width direction. - * @param yScaler scale ratio along height direction. - * @param xOffset offset along width direction in pixels. - * @param yOffset offset along height direction in pixels. - * @return transformed BBox. - */ -BBox TransformBox(const BBox &bbox, float xScaler, float yScaler, float xOffset, float yOffset); - -/** - * Squarify BBox. - * @param box input BBox. - * @param boundary boundary BBox used for clamping. - * @param scale scaling factor. - * @return squarified BBox. - */ -BBox SquarifyBox(const BBox &box, const BBox &boundary, float scale); - -}} // namespace cvcore::tensor_ops - -#endif // CVCORE_BBOX_UTILS_H \ No newline at end of file diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/tensor_ops/DBScan.h b/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/tensor_ops/DBScan.h deleted file mode 100644 index 24b4dc4..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/tensor_ops/DBScan.h +++ /dev/null @@ -1,91 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2020-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -#ifndef CVCORE_DBSCAN_H -#define CVCORE_DBSCAN_H - -#include "cv/core/Array.h" -#include "cv/core/BBox.h" - -namespace cvcore { namespace tensor_ops { - -/** - * An enum. - * Enum type for BBox merge type. - */ -enum BBoxMergeType -{ - MAXIMUM, /**< merge by expanding bounding boxes */ - WEIGHTED, /**< weighted by confidence scores. */ -}; - -/** - * DBScan implementation used for post-processing of object detection. - */ -class DBScan -{ -public: - /** - * DBScan constructor. - * @param pointsize size of the input array. - * @param minPoints minimum number of neighbors within the radius. - * @param epsilon radius of neighborhood around a point. - */ - DBScan(int pointsSize, int minPoints, float epsilon); - - /** - * Run DBScan cluster and return the cluster indices. - * @param input input unclustered BBoxes array. - * @param clusters output array containing cluster ids. - */ - void doCluster(Array &input, Array &clusters); - - /** - * Run DBScan cluster and return the cluster bboxes. - * @param input input unclustered BBoxes array. - * @param output output clustered BBoxes array. - * @param type bbox merge type - */ - void doClusterAndMerge(Array &input, Array &output, BBoxMergeType type = MAXIMUM); - - /** - * Run DBScan cluster and return the cluster bboxes weighted on input weights. - * @param input input unclustered BBoxes array. - * @param weights weights needed for merging clusterd bboxes. - * @param output output clustered BBoxes array. - * @param type bbox merge type - */ - void doClusterAndMerge(Array &input, Array &weights, Array &output, - BBoxMergeType type = WEIGHTED); - - /** - * Get the number of clusters. - * @return number of clusters. - */ - int getNumClusters() const; - -private: - int m_pointsSize; - int m_numClusters; - int m_minPoints; - float m_epsilon; - Array m_clusterStates; -}; - -}} // namespace cvcore::tensor_ops - -#endif // CVCORE_DBSCAN_H diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/tensor_ops/Errors.h b/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/tensor_ops/Errors.h deleted file mode 100644 index beebce4..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/tensor_ops/Errors.h +++ /dev/null @@ -1,49 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -#ifndef CVCORE_ERRORS_H -#define CVCORE_ERRORS_H - -#include "cv/core/CVError.h" - -namespace cvcore { namespace tensor_ops { - -// TODO: Add doxygen style comments for each of the error codes -enum class TensorOpsErrorCode : std::int32_t -{ - SUCCESS = 0, - COMPUTE_ENGINE_UNSUPPORTED_BY_CONTEXT, - CAMERA_DISTORTION_MODEL_UNSUPPORTED -}; - -}} // namespace cvcore::tensor_ops - -// WARNING: Extending base C++ namespace to cover cvcore error codes -namespace std { - -template <> -struct is_error_code_enum : true_type {}; - -} // namespace std - -namespace cvcore { namespace tensor_ops { - -std::error_code make_error_code(TensorOpsErrorCode) noexcept; - -}} // namespace cvcore::tensor_ops - -#endif // CVCORE_ERRORS_H diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/tensor_ops/IImageWarp.h b/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/tensor_ops/IImageWarp.h deleted file mode 100644 index 4a81e1a..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/tensor_ops/IImageWarp.h +++ /dev/null @@ -1,63 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -#ifndef CVCORE_IIMAGEWARP_H -#define CVCORE_IIMAGEWARP_H - -#include -#include -#include - -namespace cvcore { namespace tensor_ops { - -struct ImageGrid -{ - static constexpr std::size_t MAX_HORIZ_REGIONS = 4; - static constexpr std::size_t MAX_VERT_REGIONS = 4; - static constexpr std::size_t MIN_REGION_WIDTH = 64; - static constexpr std::size_t MIN_REGION_HIGHT = 16; - - std::int8_t numHorizRegions{0}; - std::int8_t numVertRegions{0}; - std::array horizInterval; - std::array vertInterval; - std::array regionHeight; - std::array regionWidth; -}; - -class IImageWarp -{ - public: - // Public Destructor - virtual ~IImageWarp() = 0; - - protected: - // Protected Constructor(s) - IImageWarp() = default; - IImageWarp(const IImageWarp &) = default; - IImageWarp(IImageWarp &&) noexcept = default; - - // Protected Operator(s) - IImageWarp &operator=(const IImageWarp &) = default; - IImageWarp &operator=(IImageWarp &&) noexcept = default; -}; - -using ImageWarp = IImageWarp*; - -}} // namespace cvcore::tensor_ops - -#endif // CVCORE_IIMAGEWARP_H diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/tensor_ops/ITensorOperatorContext.h b/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/tensor_ops/ITensorOperatorContext.h deleted file mode 100644 index 5aa6cdb..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/tensor_ops/ITensorOperatorContext.h +++ /dev/null @@ -1,65 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -#ifndef CVCORE_ITENSOROPERATORCONTEXT_H -#define CVCORE_ITENSOROPERATORCONTEXT_H - -#include - -#include "ITensorOperatorStream.h" -#include "cv/core/CVError.h" -#include "cv/core/ComputeEngine.h" - -namespace cvcore { namespace tensor_ops { - -enum class TensorBackend : std::uint8_t -{ - NPP, - VPI, - DALI -}; - -class ITensorOperatorContext -{ -public: - // Public Constructor(s)/Destructor - virtual ~ITensorOperatorContext() noexcept = default; - - // Public Accessor Method(s) - virtual std::error_code CreateStream(TensorOperatorStream &, const ComputeEngine &) = 0; - virtual std::error_code DestroyStream(TensorOperatorStream &) = 0; - - virtual bool IsComputeEngineCompatible(const ComputeEngine &) const noexcept = 0; - - virtual TensorBackend Backend() const noexcept = 0; - -protected: - // Protected Constructor(s) - ITensorOperatorContext() = default; - ITensorOperatorContext(const ITensorOperatorContext &) = default; - ITensorOperatorContext(ITensorOperatorContext &&) noexcept = default; - - // Protected Operator(s) - ITensorOperatorContext &operator=(const ITensorOperatorContext &) = default; - ITensorOperatorContext &operator=(ITensorOperatorContext &&) noexcept = default; -}; - -using TensorOperatorContext = ITensorOperatorContext *; - -}} // namespace cvcore::tensor_ops - -#endif // CVCORE_ITENSOROPERATORCONTEXT_H diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/tensor_ops/ITensorOperatorStream.h b/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/tensor_ops/ITensorOperatorStream.h deleted file mode 100644 index c4de9df..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/tensor_ops/ITensorOperatorStream.h +++ /dev/null @@ -1,251 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -#ifndef CVCORE_ITENSOROPERATORSTREAM_H -#define CVCORE_ITENSOROPERATORSTREAM_H - -#include -#include -#include -#include - -#include "cv/core/CameraModel.h" -#include "cv/core/ComputeEngine.h" -#include "cv/core/Image.h" - -#include "Errors.h" -#include "IImageWarp.h" -#include "ImageUtils.h" -namespace cvcore { namespace tensor_ops { - -class NotImplementedException : public std::logic_error -{ -public: - NotImplementedException() - : std::logic_error("Method not yet implemented.") - { - } -}; - -class ITensorOperatorStream -{ -public: - // Public Constructor(s)/Destructor - virtual ~ITensorOperatorStream() noexcept = default; - - // Public Accessor Method(s) - virtual std::error_code Status() noexcept = 0; - - virtual std::error_code GenerateWarpFromCameraModel(ImageWarp & warp, - const ImageGrid & grid, - const CameraModel & source, - const CameraIntrinsics & target) = 0; - - virtual std::error_code DestroyWarp(ImageWarp & warp) noexcept = 0; - - // Public Mutator Method(s) - virtual std::error_code Remap(Image &outputImage, const Image &inputImage, - const ImageWarp warp, - InterpolationType interpolation = INTERP_LINEAR, - BorderType border = BORDER_ZERO) - { - throw NotImplementedException(); - return make_error_code(ErrorCode::NOT_IMPLEMENTED); - } - virtual std::error_code Remap(Image &outputImage, const Image &inputImage, - const ImageWarp warp, - InterpolationType interpolation = INTERP_LINEAR, - BorderType border = BORDER_ZERO) - { - throw NotImplementedException(); - return make_error_code(ErrorCode::NOT_IMPLEMENTED); - } - - virtual std::error_code Remap(Image &outputImage, const Image &inputImage, - const ImageWarp warp, - InterpolationType interpolation = INTERP_LINEAR, - BorderType border = BORDER_ZERO) - { - throw NotImplementedException(); - return make_error_code(ErrorCode::NOT_IMPLEMENTED); - } - - virtual std::error_code Remap(Image &outputImage, const Image &inputImage, - const ImageWarp warp, - InterpolationType interpolation = INTERP_LINEAR, - BorderType border = BORDER_ZERO) - { - throw NotImplementedException(); - return make_error_code(ErrorCode::NOT_IMPLEMENTED); - } - - virtual std::error_code Resize(Image &outputImage, const Image &inputImage, - InterpolationType interpolation = INTERP_LINEAR, BorderType border = BORDER_ZERO) - { - throw NotImplementedException(); - return make_error_code(ErrorCode::NOT_IMPLEMENTED); - } - virtual std::error_code Resize(Image &outputImage, const Image &inputImage, - InterpolationType interpolation = INTERP_LINEAR, BorderType border = BORDER_ZERO) - { - throw NotImplementedException(); - return make_error_code(ErrorCode::NOT_IMPLEMENTED); - } - virtual std::error_code Resize(Image &outputImage, const Image &inputImage, - InterpolationType interpolation = INTERP_LINEAR, BorderType border = BORDER_ZERO) - { - throw NotImplementedException(); - return make_error_code(ErrorCode::NOT_IMPLEMENTED); - } - virtual std::error_code Resize(Image &outputImage, const Image &inputImage, - InterpolationType interpolation = INTERP_LINEAR, BorderType border = BORDER_ZERO) - { - throw NotImplementedException(); - return make_error_code(ErrorCode::NOT_IMPLEMENTED); - } - virtual std::error_code Resize(Image &outputImage, const Image &inputImage, - InterpolationType interpolation = INTERP_LINEAR, BorderType border = BORDER_ZERO) - { - throw NotImplementedException(); - return make_error_code(ErrorCode::NOT_IMPLEMENTED); - } - - virtual std::error_code Normalize(Image &outputImage, const Image &inputImage - /* only configuration parameters */) - { - throw NotImplementedException(); - return make_error_code(ErrorCode::NOT_IMPLEMENTED); - } - virtual std::error_code Normalize(Image &outputImage, const Image &inputImage - /* only configuration parameters */) - { - throw NotImplementedException(); - return make_error_code(ErrorCode::NOT_IMPLEMENTED); - } - virtual std::error_code Normalize(Image &outputImage, const Image &inputImage - /* only configuration parameters */) - { - throw NotImplementedException(); - return make_error_code(ErrorCode::NOT_IMPLEMENTED); - } - virtual std::error_code Normalize(Image &outputImage, const Image &inputImage - /* only configuration parameters */) - { - throw NotImplementedException(); - return make_error_code(ErrorCode::NOT_IMPLEMENTED); - } - - virtual std::error_code ColorConvert(Image &outputImage, const Image &inputImage) - { - throw NotImplementedException(); - return make_error_code(ErrorCode::NOT_IMPLEMENTED); - } - - virtual std::error_code ColorConvert(Image &outputImage, const Image &inputImage) - { - throw NotImplementedException(); - return make_error_code(ErrorCode::NOT_IMPLEMENTED); - } - - virtual std::error_code ColorConvert(Image &outputImage, const Image &inputImage) - { - throw NotImplementedException(); - return make_error_code(ErrorCode::NOT_IMPLEMENTED); - } - - virtual std::error_code ColorConvert(Image &outputImage, const Image &inputImage) - { - throw NotImplementedException(); - return make_error_code(ErrorCode::NOT_IMPLEMENTED); - } - - virtual std::error_code ColorConvert(Image &outputImage, const Image &inputImage) - { - throw NotImplementedException(); - return make_error_code(ErrorCode::NOT_IMPLEMENTED); - } - - virtual std::error_code ColorConvert(Image &outputImage, const Image &inputImage) - { - throw NotImplementedException(); - return make_error_code(ErrorCode::NOT_IMPLEMENTED); - } - - virtual std::error_code ColorConvert(Image &outputImage, const Image &inputImage) - { - throw NotImplementedException(); - return make_error_code(ErrorCode::NOT_IMPLEMENTED); - } - - virtual std::error_code ColorConvert(Image &outputImage, const Image &inputImage) - { - throw NotImplementedException(); - return make_error_code(ErrorCode::NOT_IMPLEMENTED); - } - - virtual std::error_code ColorConvert(Image &outputImage, const Image &inputImage) - { - throw NotImplementedException(); - return make_error_code(ErrorCode::NOT_IMPLEMENTED); - } - - virtual std::error_code ColorConvert(Image &outputImage, const Image &inputImage) - { - throw NotImplementedException(); - return make_error_code(ErrorCode::NOT_IMPLEMENTED); - } - - virtual std::error_code StereoDisparityEstimator(Image &outputImage, const Image &inputLeftImage, - const Image &inputRightImage, size_t windowSize, - size_t maxDisparity) - { - throw NotImplementedException(); - return make_error_code(ErrorCode::NOT_IMPLEMENTED); - } - - virtual std::error_code StereoDisparityEstimator(Image &outputImage, const Image &inputLeftImage, - const Image &inputRightImage, size_t windowSize, - size_t maxDisparity) - { - throw NotImplementedException(); - return make_error_code(ErrorCode::NOT_IMPLEMENTED); - } - - virtual std::error_code StereoDisparityEstimator(Image &outputImage, const Image &inputLeftImage, - const Image &inputRightImage, size_t windowSize, - size_t maxDisparity) - { - throw NotImplementedException(); - return make_error_code(ErrorCode::NOT_IMPLEMENTED); - } - -protected: - // Protected Constructor(s) - ITensorOperatorStream() = default; - ITensorOperatorStream(const ITensorOperatorStream &) = default; - ITensorOperatorStream(ITensorOperatorStream &&) noexcept = default; - - // Protected Operator(s) - ITensorOperatorStream &operator=(const ITensorOperatorStream &) = default; - ITensorOperatorStream &operator=(ITensorOperatorStream &&) noexcept = default; -}; - -using TensorOperatorStream = ITensorOperatorStream *; - -}} // namespace cvcore::tensor_ops - -#endif // CVCORE_ITENSOROPERATORSTREAM_H diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/tensor_ops/ImageUtils.h b/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/tensor_ops/ImageUtils.h deleted file mode 100644 index e46cd42..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/tensor_ops/ImageUtils.h +++ /dev/null @@ -1,1091 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2020-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -#ifndef CVCORE_IMAGE_UTILS_H -#define CVCORE_IMAGE_UTILS_H - -#include - -#include - -#include "cv/core/BBox.h" -#include "cv/core/Tensor.h" - -namespace cvcore { namespace tensor_ops { - -/** - * An enum. - * Enum type for color conversion type. - */ -enum ColorConversionType -{ - BGR2RGB, /**< convert BGR to RGB. */ - RGB2BGR, /**< convert RGB to BGR. */ - BGR2GRAY, /**< convert BGR to Grayscale. */ - RGB2GRAY, /**< convert RGB to Grayscale. */ - GRAY2BGR, /**< convert Grayscale to BGR. */ - GRAY2RGB, /**< convert Grayscale to RGB. */ -}; - -/** - * An enum. - * Enum type for resize interpolation type. - */ -enum InterpolationType -{ - INTERP_NEAREST, /**< nearest interpolation. */ - INTERP_LINEAR, /**< linear interpolation. */ - INTERP_CUBIC_BSPLINE, /**< cubic bspline interpolation. */ - INTERP_CUBIC_CATMULLROM /**< cubic catmullrom interpolation. */ -}; - -/** - * An enum. - * Enum type for resize interpolation type. - */ -enum BorderType -{ - BORDER_ZERO, - BORDER_REPEAT, - BORDER_REVERSE, - BORDER_MIRROR -}; - -// please note the following functions all require GPU Tensors - -/** - * Image resizing for one channel HWC format uint_8 type Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param type interpolation type. - * @param keep_aspect_ratio whether to keep aspect ratio. - * @param stream specified cuda stream. - */ -void Resize(Tensor &dst, const Tensor &src, bool keep_aspect_ratio = false, - InterpolationType type = INTERP_LINEAR, cudaStream_t stream = 0); - -/** - * Image batch resizing for one channel NHWC uint_8 type Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param type interpolation type. - * @param keep_aspect_ratio whether to keep aspect ratio. - * @param stream specified cuda stream. - */ -void Resize(Tensor &dst, const Tensor &src, bool keep_aspect_ratio = false, - InterpolationType type = INTERP_LINEAR, cudaStream_t stream = 0); - -/** - * Image resizing of a region of interest for one channel HWC format uint_8 type Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param dstROI destination crop region. - * @param srcROI source crop region. - * @param type interpolation type. - * @param stream specified cuda stream. - */ -void CropAndResize(Tensor &dst, const Tensor &src, const BBox &dstROI, const BBox &srcROI, - InterpolationType type = INTERP_LINEAR, cudaStream_t stream = 0); - -/** - * Image resizing of a region of interest for one channel HWC format uint_8 type Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param srcROI source crop region. - * @param type interpolation type. - * @param stream specified cuda stream. - */ -void CropAndResize(Tensor &dst, const Tensor &src, const BBox &srcROI, - InterpolationType type = INTERP_LINEAR, cudaStream_t stream = 0); - -/** - * Image resizing for one channel HWC format uint_16 type Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param type interpolation type. - * @param keep_aspect_ratio whether to keep aspect ratio. - * @param stream specified cuda stream. - */ -void Resize(Tensor &dst, const Tensor &src, bool keep_aspect_ratio = false, - InterpolationType type = INTERP_LINEAR, cudaStream_t stream = 0); - -/** - * Image batch resizing for one channel HWC format uint_16 type Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param type interpolation type. - * @param keep_aspect_ratio whether to keep aspect ratio. - * @param stream specified cuda stream. - */ -void Resize(Tensor &dst, const Tensor &src, bool keep_aspect_ratio = false, - InterpolationType type = INTERP_LINEAR, cudaStream_t stream = 0); - -/** - * Image resizing of a region of interest for one channel HWC format uint_16 type Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param dstROI destination crop region. - * @param srcROI source crop region. - * @param type interpolation type. - * @param stream specified cuda stream. - */ -void CropAndResize(Tensor &dst, const Tensor &src, const BBox &dstROI, const BBox &srcROI, - InterpolationType type = INTERP_LINEAR, cudaStream_t stream = 0); - -/** - * Image resizing of a region of interest for one channel HWC format uint_16 type Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param srcROI source crop region. - * @param type interpolation type. - * @param stream specified cuda stream. - */ -void CropAndResize(Tensor &dst, const Tensor &src, const BBox &srcROI, - InterpolationType type = INTERP_LINEAR, cudaStream_t stream = 0); - -/** - * Image resizing for one channel HWC format FP32 type Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param type interpolation type. - * @param keep_aspect_ratio whether to keep aspect ratio. - * @param stream specified cuda stream. - */ -void Resize(Tensor &dst, const Tensor &src, bool keep_aspect_ratio = false, - InterpolationType type = INTERP_LINEAR, cudaStream_t stream = 0); - -/** - * Image batch resizing for one channel HWC format FP32 type Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param type interpolation type. - * @param keep_aspect_ratio whether to keep aspect ratio. - * @param stream specified cuda stream. - */ -void Resize(Tensor &dst, const Tensor &src, bool keep_aspect_ratio = false, - InterpolationType type = INTERP_LINEAR, cudaStream_t stream = 0); - -/** - * Image resizing of a region of interest for one channel HWC format FP32 type Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param dstROI destination crop region. - * @param srcROI source crop region. - * @param type interpolation type. - * @param stream specified cuda stream. - */ -void CropAndResize(Tensor &dst, const Tensor &src, const BBox &dstROI, const BBox &srcROI, - InterpolationType type = INTERP_LINEAR, cudaStream_t stream = 0); - -/** - * Image resizing of a region of interest for one channel HWC format FP32 type Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param srcROI source crop region. - * @param type interpolation type. - * @param stream specified cuda stream. - */ -void CropAndResize(Tensor &dst, const Tensor &src, const BBox &srcROI, - InterpolationType type = INTERP_LINEAR, cudaStream_t stream = 0); - -/** - * Image resizing for one channel CHW format uint_8 type Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param type interpolation type. - * @param keep_aspect_ratio whether to keep aspect ratio. - * @param stream specified cuda stream. - */ -void Resize(Tensor &dst, const Tensor &src, bool keep_aspect_ratio = false, - InterpolationType type = INTERP_LINEAR, cudaStream_t stream = 0); - -/** - * Image resizing of a region of interest for one channel CHW format uint_8 type Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param dstROI destination crop region. - * @param srcROI source crop region. - * @param type interpolation type. - * @param stream specified cuda stream. - */ -void CropAndResize(Tensor &dst, const Tensor &src, const BBox &dstROI, const BBox &srcROI, - InterpolationType type = INTERP_LINEAR, cudaStream_t stream = 0); - -/** - * Image resizing of a region of interest for one channel CHW format uint_8 type Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param srcROI source crop region. - * @param type interpolation type. - * @param stream specified cuda stream. - */ -void CropAndResize(Tensor &dst, const Tensor &src, const BBox &srcROI, - InterpolationType type = INTERP_LINEAR, cudaStream_t stream = 0); - -/** - * Image resizing for one channel CHW format uint_16 type Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param type interpolation type. - * @param keep_aspect_ratio whether to keep aspect ratio. - * @param stream specified cuda stream. - */ -void Resize(Tensor &dst, const Tensor &src, bool keep_aspect_ratio = false, - InterpolationType type = INTERP_LINEAR, cudaStream_t stream = 0); - -/** - * Image resizing of a region of interest for one channel CHW format uint_16 type Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param dstROI destination crop region. - * @param srcROI source crop region. - * @param type interpolation type. - * @param stream specified cuda stream. - */ -void CropAndResize(Tensor &dst, const Tensor &src, const BBox &dstROI, const BBox &srcROI, - InterpolationType type = INTERP_LINEAR, cudaStream_t stream = 0); - -/** - * Image resizing of a region of interest for one channel CHW format uint_16 type Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param srcROI source crop region. - * @param type interpolation type. - * @param stream specified cuda stream. - */ -void CropAndResize(Tensor &dst, const Tensor &src, const BBox &srcROI, - InterpolationType type = INTERP_LINEAR, cudaStream_t stream = 0); - -/** - * Image resizing for one channel CHW format FP32 type Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param type interpolation type. - * @param keep_aspect_ratio whether to keep aspect ratio. - * @param stream specified cuda stream. - */ -void Resize(Tensor &dst, const Tensor &src, bool keep_aspect_ratio = false, - InterpolationType type = INTERP_LINEAR, cudaStream_t stream = 0); - -/** - * Image resizing of a region of interest for one channel CHW format FP32 type Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param dstROI destination crop region. - * @param srcROI source crop region. - * @param type interpolation type. - * @param stream specified cuda stream. - */ -void CropAndResize(Tensor &dst, const Tensor &src, const BBox &dstROI, const BBox &srcROI, - InterpolationType type = INTERP_LINEAR, cudaStream_t stream = 0); - -/** - * Image resizing of a region of interest for one channel CHW format FP32 type Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param srcROI source crop region. - * @param type interpolation type. - * @param stream specified cuda stream. - */ -void CropAndResize(Tensor &dst, const Tensor &src, const BBox &srcROI, - InterpolationType type = INTERP_LINEAR, cudaStream_t stream = 0); - -/** - * Image resizing for three channels interleaved uint_8 type Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param type interpolation type. - * @param keep_aspect_ratio whether to keep aspect ratio. - * @param stream specified cuda stream. - */ -void Resize(Tensor &dst, const Tensor &src, bool keep_aspect_ratio = false, - InterpolationType type = INTERP_LINEAR, cudaStream_t stream = 0); - -/** - * Image batch resizing for three channels interleaved uint_8 type Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param type interpolation type. - * @param keep_aspect_ratio whether to keep aspect ratio. - * @param stream specified cuda stream. - */ -void Resize(Tensor &dst, const Tensor &src, bool keep_aspect_ratio = false, - InterpolationType type = INTERP_LINEAR, cudaStream_t stream = 0); - -/** - * Image resizing of a region of intesrest for three channel HWC format uint_8 type Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param dstROI destination crop region. - * @param srcROI source crop region. - * @param type interpolation type. - * @param stream specified cuda stream. - */ -void CropAndResize(Tensor &dst, const Tensor &src, const BBox &dstROI, const BBox &srcROI, - InterpolationType type = INTERP_LINEAR, cudaStream_t stream = 0); - -/** - * Image resizing of a region of intesrest for three channel HWC format uint_8 type Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param srcROI source crop region. - * @param type interpolation type. - * @param stream specified cuda stream. - */ -void CropAndResize(Tensor &dst, const Tensor &src, const BBox &srcROI, - InterpolationType type = INTERP_LINEAR, cudaStream_t stream = 0); - -/** - * Image resizing for three channels interleaved uint_16 type Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param type interpolation type. - * @param keep_aspect_ratio whether to keep aspect ratio. - * @param stream specified cuda stream. - */ -void Resize(Tensor &dst, const Tensor &src, bool keep_aspect_ratio = false, - InterpolationType type = INTERP_LINEAR, cudaStream_t stream = 0); - -/** - * Image batch resizing for three channels interleaved uint_16 type Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param type interpolation type. - * @param keep_aspect_ratio whether to keep aspect ratio. - * @param stream specified cuda stream. - */ -void Resize(Tensor &dst, const Tensor &src, bool keep_aspect_ratio = false, - InterpolationType type = INTERP_LINEAR, cudaStream_t stream = 0); - -/** - * Image resizing of a region of intesrest for three channel HWC format uint_16 type Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param dstROI destination crop region. - * @param srcROI source crop region. - * @param type interpolation type. - * @param stream specified cuda stream. - */ -void CropAndResize(Tensor &dst, const Tensor &src, const BBox &dstROI, const BBox &srcROI, - InterpolationType type = INTERP_LINEAR, cudaStream_t stream = 0); - -/** - * Image resizing of a region of intesrest for three channel HWC format uint_16 type Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param srcROI source crop region. - * @param type interpolation type. - * @param stream specified cuda stream. - */ -void CropAndResize(Tensor &dst, const Tensor &src, const BBox &srcROI, - InterpolationType type = INTERP_LINEAR, cudaStream_t stream = 0); - -/** - * Image resizing for three channels interleaved float type Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param type interpolation type. - * @param keep_aspect_ratio whether to keep aspect ratio. - * @param stream specified cuda stream. - */ -void Resize(Tensor &dst, const Tensor &src, bool keep_aspect_ratio = false, - InterpolationType type = INTERP_LINEAR, cudaStream_t stream = 0); - -/** - * Image batch resizing for three channels interleaved float type Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param type interpolation type. - * @param keep_aspect_ratio whether to keep aspect ratio. - * @param stream specified cuda stream. - */ -void Resize(Tensor &dst, const Tensor &src, bool keep_aspect_ratio = false, - InterpolationType type = INTERP_LINEAR, cudaStream_t stream = 0); - -/** - * Image resizing of a region of intesrest for three channel HWC format float type Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param dstROI destination crop region. - * @param srcROI source crop region. - * @param type interpolation type. - * @param stream specified cuda stream. - */ -void CropAndResize(Tensor &dst, const Tensor &src, const BBox &dstROI, const BBox &srcROI, - InterpolationType type = INTERP_LINEAR, cudaStream_t stream = 0); - -/** - * Image resizing of a region of intesrest for three channel HWC format float type Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param srcROI source crop region. - * @param type interpolation type. - * @param stream specified cuda stream. - */ -void CropAndResize(Tensor &dst, const Tensor &src, const BBox &srcROI, - InterpolationType type = INTERP_LINEAR, cudaStream_t stream = 0); - -/** - * Crop a region of interest for one channel HWC format uint_8 type Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param srcROI source crop region. - * @param stream specified cuda stream. - */ -void Crop(Tensor &dst, const Tensor &src, const BBox &srcROI, cudaStream_t stream = 0); - -/** - * Crop a region of interest for one channel HWC format uint_16 type Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param srcROI source crop region. - * @param stream specified cuda stream. - */ -void Crop(Tensor &dst, const Tensor &src, const BBox &srcROI, cudaStream_t stream = 0); - -/** - * Crop a region of interest for one channel HWC format float type Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param srcROI source crop region. - * @param stream specified cuda stream. - */ -void Crop(Tensor &dst, const Tensor &src, const BBox &srcROI, cudaStream_t stream = 0); - -/** - * Crop a region of interest for one channel CHW format uint_8 type Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param srcROI source crop region. - * @param stream specified cuda stream. - */ -void Crop(Tensor &dst, const Tensor &src, const BBox &srcROI, cudaStream_t stream = 0); - -/** - * Crop a region of interest for one channel CHW format uint_16 type Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param srcROI source crop region. - * @param stream specified cuda stream. - */ -void Crop(Tensor &dst, const Tensor &src, const BBox &srcROI, cudaStream_t stream = 0); - -/** - * Crop a region of interest for one channel CHW format float type Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param srcROI source crop region. - * @param stream specified cuda stream. - */ -void Crop(Tensor &dst, const Tensor &src, const BBox &srcROI, cudaStream_t stream = 0); - -/** - * Crop a region of interest for three channels HWC format uint_8 type Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param srcROI source crop region. - * @param stream specified cuda stream. - */ -void Crop(Tensor &dst, const Tensor &src, const BBox &srcROI, cudaStream_t stream = 0); - -/** - * Crop a region of interest for three channels HWC format uint_16 type Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param srcROI source crop region. - * @param stream specified cuda stream. - */ -void Crop(Tensor &dst, const Tensor &src, const BBox &srcROI, cudaStream_t stream = 0); - -/** - * Crop a region of interest for three channels HWC format float type Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param srcROI source crop region. - * @param stream specified cuda stream. - */ -void Crop(Tensor &dst, const Tensor &src, const BBox &srcROI, cudaStream_t stream = 0); - -/** - * Apply a perspective transformation to one channel HWC format uint_8 type Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param coeffs 3x3 transformation matrix. - * @param type interpolation type. - * @param stream specified cuda stream. - */ -void WarpPerspective(Tensor &dst, const Tensor &src, const double coeffs[3][3], - InterpolationType type = INTERP_LINEAR, cudaStream_t stream = 0); - -/** - * Apply a perspective transformation to one channel HWC format uint_16 type Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param coeffs 3x3 transformation matrix. - * @param type interpolation type. - * @param stream specified cuda stream. - */ -void WarpPerspective(Tensor &dst, const Tensor &src, const double coeffs[3][3], - InterpolationType type = INTERP_LINEAR, cudaStream_t stream = 0); - -/** - * Apply a perspective transformation to one channel HWC format float type Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param coeffs 3x3 transformation matrix. - * @param type interpolation type. - * @param stream specified cuda stream. - */ -void WarpPerspective(Tensor &dst, const Tensor &src, const double coeffs[3][3], - InterpolationType type = INTERP_LINEAR, cudaStream_t stream = 0); - -/** - * Apply a perspective transformation to three channels HWC format uint_8 type Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param coeffs 3x3 transformation matrix. - * @param type interpolation type. - * @param stream specified cuda stream. - */ -void WarpPerspective(Tensor &dst, const Tensor &src, const double coeffs[3][3], - InterpolationType type = INTERP_LINEAR, cudaStream_t stream = 0); - -/** - * Apply a perspective transformation to three channels HWC format uint_16 type Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param coeffs 3x3 transformation matrix. - * @param type interpolation type. - * @param stream specified cuda stream. - */ -void WarpPerspective(Tensor &dst, const Tensor &src, const double coeffs[3][3], - InterpolationType type = INTERP_LINEAR, cudaStream_t stream = 0); - -/** - * Apply a perspective transformation to three channels HWC format float type Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param coeffs 3x3 transformation matrix. - * @param type interpolation type. - * @param stream specified cuda stream. - */ -void WarpPerspective(Tensor &dst, const Tensor &src, const double coeffs[3][3], - InterpolationType type = INTERP_LINEAR, cudaStream_t stream = 0); - -/** Color conversion between two three channels interleaved uint_8 type Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param type color conversion type. - * @param stream specified cuda stream. - */ -void ConvertColorFormat(Tensor &dst, const Tensor &src, ColorConversionType type, - cudaStream_t stream = 0); - -/** Batch color conversion between three channels interleaved uint_8 type Tensors. - * @param dst destination tensor. - * @param src source tensor. - * @param type color conversion type. - * @param stream specified cuda stream. - */ -void ConvertColorFormat(Tensor &dst, const Tensor &src, ColorConversionType type, - cudaStream_t stream = 0); - -/** Color conversion between two three channels interleaved uint_16 type Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param type color conversion type. - * @param stream specified cuda stream. - */ -void ConvertColorFormat(Tensor &dst, const Tensor &src, ColorConversionType type, - cudaStream_t stream = 0); - -/** Batch color conversion between three channels interleaved uint_16 type Tensors. - * @param dst destination tensor. - * @param src source tensor. - * @param type color conversion type. - * @param stream specified cuda stream. - */ -void ConvertColorFormat(Tensor &dst, const Tensor &src, ColorConversionType type, - cudaStream_t stream = 0); - -/** Color conversion between two three channels interleaved float type Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param type color conversion type. - * @param stream specified cuda stream. - */ -void ConvertColorFormat(Tensor &dst, const Tensor &src, ColorConversionType type, - cudaStream_t stream = 0); - -/** Batch color conversion between three channels interleaved float type Tensors. - * @param dst destination tensor. - * @param src source tensor. - * @param type color conversion type. - * @param stream specified cuda stream. - */ -void ConvertColorFormat(Tensor &dst, const Tensor &src, ColorConversionType type, - cudaStream_t stream = 0); - -/** Color conversion from three channels interleaved uint_8 type Tensor to one channel Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param type color conversion type. - * @param stream specified cuda stream. - */ -void ConvertColorFormat(Tensor &dst, const Tensor &src, ColorConversionType type, - cudaStream_t stream = 0); - -/** Color conversion from three channels interleaved uint_16 type Tensor to one channel Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param type color conversion type. - * @param stream specified cuda stream. - */ -void ConvertColorFormat(Tensor &dst, const Tensor &src, ColorConversionType type, - cudaStream_t stream = 0); - -/** Color conversion from three channels interleaved float type Tensor to one channel Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param type color conversion type. - * @param stream specified cuda stream. - */ -void ConvertColorFormat(Tensor &dst, const Tensor &src, ColorConversionType type, - cudaStream_t stream = 0); - -/** Color conversion from one channel interleaved uint_8 type Tensor to three channels Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param type color conversion type. - * @param stream specified cuda stream. - */ -void ConvertColorFormat(Tensor &dst, const Tensor &src, ColorConversionType type, - cudaStream_t stream = 0); - -/** Color conversion from one channel interleaved uint_16 type Tensor to three channels Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param type color conversion type. - * @param stream specified cuda stream. - */ -void ConvertColorFormat(Tensor &dst, const Tensor &src, ColorConversionType type, - cudaStream_t stream = 0); - -/** Color conversion from one channel interleaved float type Tensor to three channels Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param type color conversion type. - * @param stream specified cuda stream. - */ -void ConvertColorFormat(Tensor &dst, const Tensor &src, ColorConversionType type, - cudaStream_t stream = 0); - -/** Convert bit depth from F32 to U8 of a single channel channel Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param scale multiply the pixel values by a factor. - * @param stream specified cuda stream. - */ -void ConvertBitDepth(Tensor &dst, Tensor &src, const float scale, cudaStream_t stream = 0); - -/** Convert bit depth from F32 to U8 of a N * single channel Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param scale multiply the pixel values by a factor. - * @param stream specified cuda stream. - */ -void ConvertBitDepth(Tensor &dst, Tensor &src, const float scale, cudaStream_t stream = 0); - -/** - * Normalization for three channels interleaved uint8_t type Tensor. - * Each element x will be transformed to (x + offset) * scale - * @param dst destination tensor. - * @param src source tensor. - * @param scale scaling factor for normalization. - * @param offset offset constant for normalization. - * @stream specified cuda stream. - */ -void Normalize(Tensor &dst, const Tensor &src, const float scale[3], const float offset[3], - cudaStream_t stream = 0); - -/** - * Batch normalization for three channels interleaved uint8_t type Tensor. - * Each element x will be transformed to (x + offset) * scale - * @param dst destination tensor. - * @param src source tensor. - * @param scale scaling factor for normalization. - * @param offset offset constant for normalization. - * @stream specified cuda stream. - */ -void Normalize(Tensor &dst, const Tensor &src, const float scale[3], const float offset[3], - cudaStream_t stream = 0); - -/** - * Normalization for three channels interleaved uint16_t type Tensor. - * Each element x will be transformed to (x + offset) * scale - * @param dst destination tensor. - * @param src source tensor. - * @param scale scaling factor for normalization. - * @param offset offset constant for normalization. - * @stream specified cuda stream. - */ -void Normalize(Tensor &dst, const Tensor &src, const float scale[3], const float offset[3], - cudaStream_t stream = 0); - -/** - * Batch normalization for three channels interleaved uint16_t type Tensor. - * Each element x will be transformed to (x + offset) * scale - * @param dst destination tensor. - * @param src source tensor. - * @param scale scaling factor for normalization. - * @param offset offset constant for normalization. - * @stream specified cuda stream. - */ -void Normalize(Tensor &dst, const Tensor &src, const float scale[3], - const float offset[3], cudaStream_t stream = 0); - -/** - * Normalization for three channels interleaved float type Tensor. - * Each element x will be transformed to (x + offset) * scale - * @param dst destination tensor. - * @param src source tensor. - * @param scale scaling factor for normalization. - * @param offset offset constant for normalization. - * @stream specified cuda stream. - */ -void Normalize(Tensor &dst, const Tensor &src, const float scale[3], const float offset[3], - cudaStream_t stream = 0); - -/** - * Batch normalization for three channels interleaved float type Tensor. - * Each element x will be transformed to (x + offset) * scale - * @param dst destination tensor. - * @param src source tensor. - * @param scale scaling factor for normalization. - * @param offset offset constant for normalization. - * @stream specified cuda stream. - */ -void Normalize(Tensor &dst, const Tensor &src, const float scale[3], - const float offset[3], cudaStream_t stream = 0); - -/** - * Normalization for one channel interleaved uint8_t type Tensor. - * Each element x will be transformed to (x + offset) * scale - * @param dst destination tensor. - * @param src source tensor. - * @param scale scaling factor for normalization. - * @param offset offset constant for normalization. - * @stream specified cuda stream. - */ -void Normalize(Tensor &dst, const Tensor &src, const float scale, const float offset, - cudaStream_t stream = 0); - -/** - * Batch normalization for one channel interleaved uint8_t type Tensor. - * Each element x will be transformed to (x + offset) * scale - * @param dst destination tensor. - * @param src source tensor. - * @param scale scaling factor for normalization. - * @param offset offset constant for normalization. - * @stream specified cuda stream. - */ -void Normalize(Tensor &dst, const Tensor &src, const float scale, const float offset, - cudaStream_t stream = 0); - -/** - * Normalization for one channel interleaved uint16_t type Tensor. - * Each element x will be transformed to (x + offset) * scale - * @param dst destination tensor. - * @param src source tensor. - * @param scale scaling factor for normalization. - * @param offset offset constant for normalization. - * @stream specified cuda stream. - */ -void Normalize(Tensor &dst, const Tensor &src, const float scale, const float offset, - cudaStream_t stream = 0); - -/** - * Batch normalization for one channel interleaved uint16_t type Tensor. - * Each element x will be transformed to (x + offset) * scale - * @param dst destination tensor. - * @param src source tensor. - * @param scale scaling factor for normalization. - * @param offset offset constant for normalization. - * @stream specified cuda stream. - */ -void Normalize(Tensor &dst, const Tensor &src, const float scale, const float offset, - cudaStream_t stream = 0); - -/** - * Normalization for one channel interleaved float type Tensor. - * Each element x will be transformed to (x + offset) * scale - * @param dst destination tensor. - * @param src source tensor. - * @param scale scaling factor for normalization. - * @param offset offset constant for normalization. - * @stream specified cuda stream. - */ -void Normalize(Tensor &dst, const Tensor &src, const float scale, const float offset, - cudaStream_t stream = 0); - -/** - * Batch normalization for one channel interleaved float type Tensor. - * Each element x will be transformed to (x + offset) * scale - * @param dst destination tensor. - * @param src source tensor. - * @param scale scaling factor for normalization. - * @param offset offset constant for normalization. - * @stream specified cuda stream. - */ -void Normalize(Tensor &dst, const Tensor &src, const float scale, const float offset, - cudaStream_t stream = 0); - -/** - * Normalization for one channel planar uint8_t type Tensor. - * Each element x will be transformed to (x + offset) * scale - * @param dst destination tensor. - * @param src source tensor. - * @param scale scaling factor for normalization. - * @param offset offset constant for normalization. - * @stream specified cuda stream. - */ -void Normalize(Tensor &dst, const Tensor &src, const float scale, const float offset, - cudaStream_t stream = 0); - -/** - * Batch normalization for one channel planar uint8_t type Tensor. - * Each element x will be transformed to (x + offset) * scale - * @param dst destination tensor. - * @param src source tensor. - * @param scale scaling factor for normalization. - * @param offset offset constant for normalization. - * @stream specified cuda stream. - */ -void Normalize(Tensor &dst, const Tensor &src, const float scale, const float offset, - cudaStream_t stream = 0); - -/** - * Normalization for one channel planar uint16_t type Tensor. - * Each element x will be transformed to (x + offset) * scale - * @param dst destination tensor. - * @param src source tensor. - * @param scale scaling factor for normalization. - * @param offset offset constant for normalization. - * @stream specified cuda stream. - */ -void Normalize(Tensor &dst, const Tensor &src, const float scale, const float offset, - cudaStream_t stream = 0); - -/** - * Batch normalization for one channel planar uint16_t type Tensor. - * Each element x will be transformed to (x + offset) * scale - * @param dst destination tensor. - * @param src source tensor. - * @param scale scaling factor for normalization. - * @param offset offset constant for normalization. - * @stream specified cuda stream. - */ -void Normalize(Tensor &dst, const Tensor &src, const float scale, const float offset, - cudaStream_t stream = 0); - -/** - * Normalization for one channel planar float type Tensor. - * Each element x will be transformed to (x + offset) * scale - * @param dst destination tensor. - * @param src source tensor. - * @param scale scaling factor for normalization. - * @param offset offset constant for normalization. - * @stream specified cuda stream. - */ -void Normalize(Tensor &dst, const Tensor &src, const float scale, const float offset, - cudaStream_t stream = 0); - -/** - * Batch normalization for one channel planar float type Tensor. - * Each element x will be transformed to (x + offset) * scale - * @param dst destination tensor. - * @param src source tensor. - * @param scale scaling factor for normalization. - * @param offset offset constant for normalization. - * @stream specified cuda stream. - */ -void Normalize(Tensor &dst, const Tensor &src, const float scale, const float offset, - cudaStream_t stream = 0); - -/** - * Convert interleaved image to planar image for three channels uint8_t type Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param stream specified cuda stream. - */ -void InterleavedToPlanar(Tensor &dst, const Tensor &src, cudaStream_t stream = 0); - -/** - * Batch convert interleaved image to planar image for three channels uint8_t type Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param stream specified cuda stream. - */ -void InterleavedToPlanar(Tensor &dst, const Tensor &src, cudaStream_t stream = 0); - -/** - * Convert interleaved image to planar image for three channels uint16_t type Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param stream specified cuda stream. - */ -void InterleavedToPlanar(Tensor &dst, const Tensor &src, cudaStream_t stream = 0); - -/** - * Batch Convert interleaved image to planar image for three channels uint16_t type Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param stream specified cuda stream. - */ -void InterleavedToPlanar(Tensor &dst, const Tensor &src, cudaStream_t stream = 0); - -/** - * Convert interleaved image to planar image for three channels float type Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param stream specified cuda stream. - */ -void InterleavedToPlanar(Tensor &dst, const Tensor &src, cudaStream_t stream = 0); - -/** - * Batch convert interleaved image to planar image for three channels float type Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param stream specified cuda stream. - */ -void InterleavedToPlanar(Tensor &dst, const Tensor &src, cudaStream_t stream = 0); - -/** - * Convert interleaved image to planar image for single channel uint8_t type Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param stream specified cuda stream. - */ -void InterleavedToPlanar(Tensor &dst, const Tensor &src, cudaStream_t stream = 0); - -/** - * Batch convert interleaved image to planar image for single channel uint8_t type Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param stream specified cuda stream. - */ -void InterleavedToPlanar(Tensor &dst, const Tensor &src, cudaStream_t stream = 0); - -/** - * Convert interleaved image to planar image for single channel uint16_t type Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param stream specified cuda stream. - */ -void InterleavedToPlanar(Tensor &dst, const Tensor &src, cudaStream_t stream = 0); - -/** - * Batch Convert interleaved image to planar image for single channel uint16_t type Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param stream specified cuda stream. - */ -void InterleavedToPlanar(Tensor &dst, const Tensor &src, cudaStream_t stream = 0); - -/** - * Convert interleaved image to planar image for single channel float type Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param stream specified cuda stream. - */ -void InterleavedToPlanar(Tensor &dst, const Tensor &src, cudaStream_t stream = 0); - -/** - * Batch convert interleaved image to planar image for single channel float type Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param stream specified cuda stream. - */ -void InterleavedToPlanar(Tensor &dst, const Tensor &src, cudaStream_t stream = 0); - -/** - * Combines various functions to imitate OpenCV blobFromImage() for various type tensor input. - * @tparam TL_IN input tensor layout (HWC/NHWC). - * @tparam TL_OUT output tensor layout(CHW/NCHW). - * @tparam CC channel count. - * @tparam CT channel type for input tensor (output is fixed: F32). - */ -template -class ImageToNormalizedPlanarTensorOperator -{ -public: - /** - * Implementation for ImageToNormalizedPlanarTensorOperator. - */ - struct ImageToNormalizedPlanarTensorOperatorImpl; - - /** - * Constructor for HWC -> CHW tensors. - */ - template::type * = nullptr> - ImageToNormalizedPlanarTensorOperator(int width, int height); - - /** - * Constructor for NHWC -> NCHW tensors. - */ - template::type * = nullptr> - ImageToNormalizedPlanarTensorOperator(int width, int height, int depth); - - /** - * Destructor for ImageToNormalizedPlanarTensorOperator. - */ - ~ImageToNormalizedPlanarTensorOperator(); - - /** - * Run the composite operations on three channels tensors. - * @param dst destination tensor. - * @param src source tensor. - * @param scale scale factor for normalization. - * @param offset offset constant for normalization. - * @param swapRB whether to swap the first and last channels. - * @param keep_aspect_ratio whether to keep aspect ratio when resizing. - * @param stream specified cuda stream. - */ - template::type * = nullptr> - void operator()(Tensor &dst, const Tensor &src, const float scale[3], - const float offset[3], bool swapRB, bool keep_aspect_ratio = false, cudaStream_t stream = 0); - - /** - * Run the composite operations on single channel tensors. - * @param dst destination tensor. - * @param src source tensor. - * @param scale scale factor for normalization. - * @param offset offset constant for normalization. - * @param keep_aspect_ratio whether to keep aspect ratio when resizing. - * @param stream specified cuda stream. - */ - template::type * = nullptr> - void operator()(Tensor &dst, const Tensor &src, float scale, float offset, - bool keep_aspect_ratio = false, cudaStream_t stream = 0); - -private: - std::unique_ptr m_pImpl; -}; - -}} // namespace cvcore::tensor_ops - -#endif // CVCORE_IMAGE_UTILS_H diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/tensor_ops/NppUtils.cpp b/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/tensor_ops/NppUtils.cpp deleted file mode 100644 index 4c7cf7f..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/tensor_ops/NppUtils.cpp +++ /dev/null @@ -1,116 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2020-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -#include "NppUtils.h" - -#include -#include -#include -#include - -namespace cvcore { namespace tensor_ops { - -constexpr size_t CACHE_SIZE = 20; -static size_t timestamp = 0; -std::mutex lock; - -namespace { - -// This function involves GPU query and can be really slow -void SetupNppStreamContext(NppStreamContext &context, cudaStream_t stream) -{ - context.hStream = stream; - cudaError_t error = cudaGetDevice(&context.nCudaDeviceId); - if (error != cudaSuccess) - { - throw std::runtime_error("no devices supporting CUDA"); - } - error = cudaStreamGetFlags(context.hStream, &context.nStreamFlags); - if (error != cudaSuccess) - { - throw std::runtime_error("failed to get cuda stream flags"); - } - - cudaDeviceProp deviceProp; - error = cudaGetDeviceProperties(&deviceProp, context.nCudaDeviceId); - if (error != cudaSuccess) - { - throw std::runtime_error("no device properties"); - } - - context.nSharedMemPerBlock = deviceProp.sharedMemPerBlock; - context.nMaxThreadsPerBlock = deviceProp.maxThreadsPerBlock; - context.nMultiProcessorCount = deviceProp.multiProcessorCount; - context.nMaxThreadsPerMultiProcessor = deviceProp.maxThreadsPerMultiProcessor; - - // Refer - https://gitlab-master.nvidia.com/cv/core-modules/tensor_ops/-/merge_requests/48#note_6602087 - context.nReserved0 = 0; - - error = cudaDeviceGetAttribute(&(context.nCudaDevAttrComputeCapabilityMajor), cudaDevAttrComputeCapabilityMajor, - context.nCudaDeviceId); - if (error != cudaSuccess) - { - throw std::runtime_error("no device attribute - nCudaDevAttrComputeCapabilityMajor"); - } - - error = cudaDeviceGetAttribute(&(context.nCudaDevAttrComputeCapabilityMinor), cudaDevAttrComputeCapabilityMinor, - context.nCudaDeviceId); - if (error != cudaSuccess) - { - throw std::runtime_error("no device attribute - nCudaDevAttrComputeCapabilityMinor"); - } -} - -} // anonymous namespace - -struct Context -{ - NppStreamContext nppContext; - size_t time = 0; -}; - -NppStreamContext GetNppStreamContext(cudaStream_t stream) -{ - // Create a memory cache, all timestamp would be initialzed to 0 automatically - static std::array contextCache = {}; - - // Lock the thread - std::lock_guard guard(lock); - - size_t minTimestamp = contextCache[0].time; - size_t minIdx = 0; - for (size_t i = 0; i < CACHE_SIZE; i++) - { - auto &it = contextCache[i]; - if (it.time > 0 && it.nppContext.hStream == stream) - { - it.time = ++timestamp; - return it.nppContext; - } - if (it.time < minTimestamp) - { - minTimestamp = it.time; - minIdx = i; - } - } - auto &it = contextCache[minIdx]; - SetupNppStreamContext(it.nppContext, stream); - it.time = ++timestamp; - return it.nppContext; -} - -}} // namespace cvcore::tensor_ops diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/tensor_ops/NppUtils.h b/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/tensor_ops/NppUtils.h deleted file mode 100644 index 398ef8f..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/tensor_ops/NppUtils.h +++ /dev/null @@ -1,31 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2020-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -#ifndef CVCORE_NPP_UTILS_H -#define CVCORE_NPP_UTILS_H - -#include - -#include - -namespace cvcore { namespace tensor_ops { - -NppStreamContext GetNppStreamContext(cudaStream_t stream); - -}} // namespace cvcore::tensor_ops - -#endif // CVCORE_NPP_UTILS_H diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/tensor_ops/OneEuroFilter.h b/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/tensor_ops/OneEuroFilter.h deleted file mode 100644 index 4947042..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/tensor_ops/OneEuroFilter.h +++ /dev/null @@ -1,82 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -#ifndef CVCORE_ONEEUROFILTER_H -#define CVCORE_ONEEUROFILTER_H - -#include -#include - -#include "cv/core/CVError.h" - -namespace cvcore { namespace tensor_ops { - -/** - * Euro Filter Parameters - */ -struct OneEuroFilterParams -{ - float dataUpdateRate; /**< Data Update rate in Hz. */ - float minCutoffFreq; /**< Minimum Cut off frequency in Hz. */ - float cutoffSlope; /**< Beta or Speed coefficient which is a tuning parameter. */ - float derivCutoffFreq; /**< Cutoff frequency for derivative. */ -}; - -/* -The one euro filter is a low pass filter for filtering noisy signals in real-time. The filtering uses exponential smoothing where the smooting factor is computed dynamically using the input data update rate. The smoothing factor provides a trade off between slow speed jitter vs high speed lag. -There are two main tuning parameters for the filter, the speed coeffcient Beta and the minimum cut off frequency. -If high speed lag is a problem, increase beta; if slow speed jitter is a problem, decrease fcmin. -Reference : http://cristal.univ-lille.fr/~casiez/1euro/ -*/ -template -class OneEuroFilter -{ -public: - struct OneEuroFilterImpl; - /** - * Euro Filter Constructor. - * @param filterParams Filter parameters - */ - OneEuroFilter(const OneEuroFilterParams &filterParams); - - /** - * Reset Euro filter Parameters. - * @param filterParams Filter parameters - * @return Error code - */ - std::error_code resetParams(const OneEuroFilterParams &filterParams); - - /** - * Filter the input. Supports float, double, vector2f, Vector3f, Vector3d, Vector3f input types. - * @param inValue Noisy input to be filtered. - * @param filteredValue Filtered output - * @return Error code - */ - std::error_code execute(T &filteredValue, T inValue); - - ~OneEuroFilter(); - -private: - /** - * Implementation of EuroFilter. - */ - std::unique_ptr m_pImpl; -}; - -}} // namespace cvcore::tensor_ops - -#endif // CVCORE_ONEEUROFILTER_H diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/tensor_ops/TensorOperators.h b/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/tensor_ops/TensorOperators.h deleted file mode 100644 index e32a3df..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/tensor_ops/TensorOperators.h +++ /dev/null @@ -1,48 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -#ifndef CVCORE_TENSOROPERATORS_H -#define CVCORE_TENSOROPERATORS_H - -#include -#include -#include - -#include "cv/tensor_ops/ITensorOperatorContext.h" -#include "cv/tensor_ops/ITensorOperatorStream.h" - -namespace cvcore { namespace tensor_ops { - -class TensorContextFactory -{ -public: - static std::error_code CreateContext(TensorOperatorContext &, TensorBackend backend); - static std::error_code DestroyContext(TensorOperatorContext &context); - - static bool IsBackendSupported(TensorBackend backend); - -private: - using MultitonType = std::unordered_map>>; - - static MultitonType instances; - static std::mutex instanceMutex; -}; - -}} // namespace cvcore::tensor_ops - -#endif // CVCORE_TENSOROPERATORS_H diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/trtbackend/TRTBackend.h b/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/trtbackend/TRTBackend.h deleted file mode 100644 index 52b2e40..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/trtbackend/TRTBackend.h +++ /dev/null @@ -1,203 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2020-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -#ifndef CVCORE_TRTBACKEND_H -#define CVCORE_TRTBACKEND_H - -#include -#include -#include -#include -#include - -#include - -namespace cvcore { - -/** - * Enum to specify different precision types. - */ -enum TRTBackendPrecision -{ - INT8, /**< 8-bit integer precision. */ - FP16, /**< 16-bit float precision. */ - FP32 /**< 32-bit float precision. */ -}; - -/** - * Enum to specify TensorRT blob layout type. - */ -enum TRTBackendBlobLayout -{ - PLANAR, /**< planar input type. */ - INTERLEAVED /**< interleaved input type. */ -}; - -/** - * Struct to store TensorRT blob dimensions. - */ -struct TRTBackendBlobSize -{ - int channels; /**< channels count. */ - int height; /**< blob height. */ - int width; /**< blob width. */ -}; - -/** - * Enum to specify model type. - */ -enum ModelType -{ - ONNX, /**< ONNX model. */ - UFF, /**< Uff model. */ - TRT_ENGINE, /**< serialized TensorRT engine. */ - TRT_ENGINE_IN_MEMORY /**< a memory pointer to deserialized TensorRT ICudaEngine. */ -}; - -/** - * Parameters for TRTBackend. - */ -struct TRTBackendParams -{ - // This constructor is for backward compatibility with all the other core modules which use trtbackend. - TRTBackendParams(std::vector inputLayers, std::vector inputDims, - std::vector outputLayers, std::string weightsPath, std::string enginePath, - TRTBackendPrecision precision, int batchSize, ModelType modelType, bool explicitBatch, - void *engine = nullptr, TRTBackendBlobLayout layout = PLANAR) - : inputLayers(inputLayers) - , inputDims(inputDims) - , outputLayers(outputLayers) - , weightsPath(weightsPath) - , enginePath(enginePath) - , precision(precision) - , batchSize(batchSize) - , modelType(modelType) - , explicitBatch(explicitBatch) - , trtEngineInMemory(engine) - , inputLayout(layout) - { - } - - std::vector inputLayers; /**< names of input layers. */ - std::vector inputDims; /**< dimensions of input layers. */ - std::vector outputLayers; /**< names of output layers. */ - std::string weightsPath; /**< model weight path. */ - std::string enginePath; /**< TensorRT engine path. */ - TRTBackendPrecision precision; /**< TensorRT engine precision. */ - int batchSize; /**< inference batch size. */ - ModelType modelType; /**< model type. */ - bool explicitBatch; /**< whether it is explicit batch dimension. */ - void * - trtEngineInMemory; /**< pointer to hold deserialized TensorRT ICudaEngine, for ModelType::TRT_ENGINE_IN_MEMORY. */ - TRTBackendBlobLayout inputLayout; /**< input blob layout. */ -}; - -// Forward declaration -struct TRTImpl; - -/** - * TensorRT wrapper class. - */ -class TRTBackend -{ -public: - /** - * Constructor of TRTBackend. - * @param modelFilePath path of the network model. - * @param precision TensorRT precision type. - */ - TRTBackend(const char *modelFilePath, TRTBackendPrecision precision, int batchSize = 1, bool explicitBatch = false); - - /** - * Constructor of TRTBackend. - * @param inputParams parameters of TRTBackend. - */ - TRTBackend(TRTBackendParams &inputParams); - - /** - * Destructor of TRTBackend. - */ - ~TRTBackend(); - - /** - * Run inference. - * @param buffer input GPU buffers. - */ - [[deprecated]] void infer(void **buffer); - - /** - * Run inference. - * @param buffer input GPU buffers. - * @param batchSize run infer with specific batch size, passed in setBindingDimension() call. - * @param stream update cuda stream in this instance. - */ - void infer(void **buffer, int batchSize, cudaStream_t stream); - - /** - * Get the cuda stream for TRTBackend. - * @return return cuda stream ID. - */ - [[deprecated]] cudaStream_t getCUDAStream() const; - - /** - * Set the cuda stream for TRTBackend. - * @param stream specified cudaStream_t for the TensorRT Engine. - */ - [[deprecated]] void setCUDAStream(cudaStream_t stream); - - /** - * Get all input/output bindings count. - * @return number of all bindings. - */ - int getBlobCount() const; - - /** - * Get the blob dimension for given blob index. - * @param blobIndex blob index. - * @return blob dimension for the given index. - */ - TRTBackendBlobSize getTRTBackendBlobSize(int blobIndex) const; - - /** - * Get the total number of elements for the given blob index. - * @param blobIndex blob index. - * @return total size for the given index. - */ - int getBlobLinearSize(int blobIndex) const; - - /** - * Get the blob index for the given blob name. - * @param blobName blob name. - * @return blob index for the given name. - */ - int getBlobIndex(const char *blobName) const; - - /** - * Check if binding is input. - * @param index binding index. - * @return whether the binding is input. - */ - bool bindingIsInput(const int index) const; - -private: - // TRT related variables - std::unique_ptr m_pImpl; -}; - -} // namespace cvcore - -#endif // CVCORE_TRTBACKEND_H diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/core/cvcore/Array.cpp b/isaac_ros_bi3d/gxf/bi3d/cvcore/src/core/cvcore/Array.cpp deleted file mode 100644 index 05cd535..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/core/cvcore/Array.cpp +++ /dev/null @@ -1,145 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2020-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -#include "cv/core/Array.h" - -#include - -#include -#include - -namespace cvcore { - -ArrayBase::ArrayBase() - : m_data{nullptr} - , m_size{0} - , m_capacity{0} - , m_elemSize{0} - , m_isOwning{false} - , m_isCPU{true} -{ -} - -ArrayBase::ArrayBase(std::size_t capacity, std::size_t elemSize, void *dataPtr, bool isCPU) - : ArrayBase() -{ - m_isOwning = false; - m_isCPU = isCPU; - m_elemSize = elemSize; - m_capacity = capacity; - m_data = dataPtr; -} - -ArrayBase::ArrayBase(std::size_t capacity, std::size_t elemSize, bool isCPU) - : ArrayBase(capacity, elemSize, nullptr, isCPU) -{ - m_isOwning = true; - - // allocate - const size_t arraySize = capacity * elemSize; - if (arraySize > 0) - { - if (isCPU) - { - m_data = std::malloc(arraySize); - } - else - { - if (cudaMalloc(&m_data, arraySize) != 0) - { - throw std::runtime_error("CUDA alloc() failed!"); - } - } - } -} - -ArrayBase::~ArrayBase() -{ - if (m_isOwning) - { - if (m_isCPU) - { - std::free(m_data); - } - else - { - cudaFree(m_data); - } - } -} - -ArrayBase::ArrayBase(ArrayBase &&t) - : ArrayBase() -{ - *this = std::move(t); -} - -ArrayBase &ArrayBase::operator=(ArrayBase &&t) -{ - using std::swap; - - swap(m_data, t.m_data); - swap(m_size, t.m_size); - swap(m_capacity, t.m_capacity); - swap(m_elemSize, t.m_elemSize); - swap(m_isOwning, t.m_isOwning); - swap(m_isCPU, t.m_isCPU); - - return *this; -} - -void *ArrayBase::getElement(int idx) const -{ - assert(idx >= 0 && idx < m_capacity); - return reinterpret_cast(m_data) + idx * m_elemSize; -} - -std::size_t ArrayBase::getSize() const -{ - return m_size; -} -std::size_t ArrayBase::getCapacity() const -{ - return m_capacity; -} -std::size_t ArrayBase::getElementSize() const -{ - return m_elemSize; -} - -void ArrayBase::setSize(std::size_t size) -{ - assert(size <= m_capacity); - m_size = size; -} - -bool ArrayBase::isCPU() const -{ - return m_isCPU; -} - -bool ArrayBase::isOwning() const -{ - return m_isOwning; -} - -void *ArrayBase::getData() const -{ - return m_data; -} - -} // namespace cvcore diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/core/cvcore/Dummy.cu b/isaac_ros_bi3d/gxf/bi3d/cvcore/src/core/cvcore/Dummy.cu deleted file mode 100644 index e69de29..0000000 diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/core/cvcore/MathTypes.cpp b/isaac_ros_bi3d/gxf/bi3d/cvcore/src/core/cvcore/MathTypes.cpp deleted file mode 100644 index e0b4adc..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/core/cvcore/MathTypes.cpp +++ /dev/null @@ -1,244 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2020-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -#include "cv/core/MathTypes.h" - -namespace cvcore { - -namespace { - -AxisAngleRotation RotationMatrixToAxisAngleUtil(const std::vector &rotMatrix) -{ - assert(rotMatrix.size() == 9); - AxisAngleRotation axisangle; - double row0 = 0.5 * (rotMatrix[7] - rotMatrix[5]); - double row1 = 0.5 * (rotMatrix[2] - rotMatrix[6]); - double row2 = 0.5 * (rotMatrix[3] - rotMatrix[1]); - - double sin_angle = std::sqrt(row0 * row0 + row1 * row1 + row2 * row2); - double cos_angle = 0.5 * (rotMatrix[0] + rotMatrix[4] + rotMatrix[8] - 1.0); - sin_angle = sin_angle > 1.0 ? 1.0 : sin_angle; - cos_angle = cos_angle > 1.0 ? 1.0 : (cos_angle < -1.0 ? -1.0 : cos_angle); - - if (sin_angle == 0.0) - { - axisangle.angle = 0; - axisangle.axis.x = 0; - axisangle.axis.y = 0; - axisangle.axis.z = 0; - } - else - { - axisangle.angle = std::atan2(sin_angle, cos_angle); - axisangle.axis.x = row0 / sin_angle; - axisangle.axis.y = row1 / sin_angle; - axisangle.axis.z = row2 / sin_angle; - } - return axisangle; -} -} // namespace - -Vector3d RotationMatrixToRotationVector(const std::vector &rotMatrix) -{ - AxisAngleRotation axisangle = RotationMatrixToAxisAngleUtil(rotMatrix); - Vector3d rotVector; - rotVector.x = axisangle.angle * axisangle.axis.x; - rotVector.y = axisangle.angle * axisangle.axis.y; - rotVector.z = axisangle.angle * axisangle.axis.z; - return rotVector; -} - -AxisAngleRotation RotationMatrixToAxisAngleRotation(const std::vector &rotMatrix) -{ - AxisAngleRotation axisangle = RotationMatrixToAxisAngleUtil(rotMatrix); - return axisangle; -} - -std::vector AxisAngleToRotationMatrix(const AxisAngleRotation &axisangle) -{ - std::vector rotMatrix; - rotMatrix.resize(9); - double cosangle = std::cos(axisangle.angle); - double sinagle = std::sin(axisangle.angle); - double temp = 1.0 - cosangle; - - rotMatrix[0] = cosangle + axisangle.axis.x * axisangle.axis.x * temp; - rotMatrix[4] = cosangle + axisangle.axis.y * axisangle.axis.y * temp; - rotMatrix[8] = cosangle + axisangle.axis.z * axisangle.axis.z * temp; - - double value1 = axisangle.axis.x * axisangle.axis.y * temp; - double value2 = axisangle.axis.z * sinagle; - rotMatrix[3] = value1 + value2; - rotMatrix[1] = value1 - value2; - value1 = axisangle.axis.x * axisangle.axis.z * temp; - value2 = axisangle.axis.y * sinagle; - rotMatrix[6] = value1 - value2; - rotMatrix[2] = value1 + value2; - value1 = axisangle.axis.y * axisangle.axis.z * temp; - value2 = axisangle.axis.x * sinagle; - rotMatrix[7] = value1 + value2; - rotMatrix[5] = value1 - value2; - return rotMatrix; -} - -Vector3d AxisAngleRotationToRotationVector(const AxisAngleRotation &axisangle) -{ - double angle = axisangle.angle; - Vector3d rotVector; - rotVector.x = angle * axisangle.axis.x; - rotVector.y = angle * axisangle.axis.y; - rotVector.z = angle * axisangle.axis.z; - return rotVector; -} - -AxisAngleRotation RotationVectorToAxisAngleRotation(const Vector3d &rotVector) -{ - double normVector = rotVector.x * rotVector.x + rotVector.y * rotVector.y + rotVector.z * rotVector.z; - normVector = std::sqrt(normVector); - AxisAngleRotation axisangle; - if (normVector == 0) - { - axisangle.angle = 0; - axisangle.axis.x = 0; - axisangle.axis.y = 0; - axisangle.axis.z = 0; - } - else - { - axisangle.angle = normVector; - axisangle.axis.x = rotVector.x / normVector; - axisangle.axis.y = rotVector.y / normVector; - axisangle.axis.z = rotVector.z / normVector; - } - return axisangle; -} - -Quaternion AxisAngleRotationToQuaternion(const AxisAngleRotation &axisangle) -{ - Quaternion qrotation; - qrotation.qx = axisangle.axis.x * sin(axisangle.angle / 2); - qrotation.qy = axisangle.axis.y * sin(axisangle.angle / 2); - qrotation.qz = axisangle.axis.z * sin(axisangle.angle / 2); - qrotation.qw = std::cos(axisangle.angle / 2); - return qrotation; -} - -AxisAngleRotation QuaternionToAxisAngleRotation(const Quaternion &qrotation) -{ - Quaternion qtemp(qrotation.qx, qrotation.qy, qrotation.qz, qrotation.qw); - if (qrotation.qw > 1) - { - double qnorm = qrotation.qx * qrotation.qx + qrotation.qy * qrotation.qy + qrotation.qz * qrotation.qz + - qrotation.qw * qrotation.qw; - qnorm = std::sqrt(qnorm); - qtemp.qx = qrotation.qx / qnorm; - qtemp.qy = qrotation.qy / qnorm; - qtemp.qz = qrotation.qz / qnorm; - qtemp.qw = qrotation.qw / qnorm; - } - AxisAngleRotation axisangle; - axisangle.angle = 2 * std::acos(qtemp.qw); - double normaxis = std::sqrt(1 - qtemp.qw * qtemp.qw); - if (normaxis < 0.001) - { - axisangle.axis.x = qtemp.qx; - axisangle.axis.y = qtemp.qy; - axisangle.axis.z = qtemp.qz; - } - else - { - axisangle.axis.x = qtemp.qx / normaxis; - axisangle.axis.y = qtemp.qy / normaxis; - axisangle.axis.z = qtemp.qz / normaxis; - } - return axisangle; -} - -std::vector QuaternionToRotationMatrix(const Quaternion &qrotation) -{ - std::vector rotMatrix; - rotMatrix.resize(9); - double qxsquare = qrotation.qx * qrotation.qx; - double qysquare = qrotation.qy * qrotation.qy; - double qzsquare = qrotation.qz * qrotation.qz; - double qwsquare = qrotation.qw * qrotation.qw; - - // Ensure quaternion is normalized - double invsersenorm = 1 / (qxsquare + qysquare + qzsquare + qwsquare); - rotMatrix[0] = (qxsquare - qysquare - qzsquare + qwsquare) * invsersenorm; - rotMatrix[4] = (-qxsquare + qysquare - qzsquare + qwsquare) * invsersenorm; - rotMatrix[8] = (-qxsquare - qysquare + qzsquare + qwsquare) * invsersenorm; - - double value1 = qrotation.qx * qrotation.qy; - double value2 = qrotation.qz * qrotation.qw; - rotMatrix[3] = 2.0 * (value1 + value2) * invsersenorm; - rotMatrix[1] = 2.0 * (value1 - value2) * invsersenorm; - - value1 = qrotation.qx * qrotation.qz; - value2 = qrotation.qy * qrotation.qw; - rotMatrix[6] = 2.0 * (value1 - value2) * invsersenorm; - rotMatrix[2] = 2.0 * (value1 + value2) * invsersenorm; - value1 = qrotation.qz * qrotation.qy; - value2 = qrotation.qx * qrotation.qw; - rotMatrix[7] = 2.0 * (value1 + value2) * invsersenorm; - rotMatrix[5] = 2.0 * (value1 - value2) * invsersenorm; - return rotMatrix; -} - -Quaternion RotationMatrixToQuaternion(const std::vector &rotMatrix) -{ - Quaternion qrotation; - double diagsum = rotMatrix[0] + rotMatrix[4] + rotMatrix[8]; - if (diagsum > 0) - { - double temp = 1 / (2 * std::sqrt(diagsum + 1.0)); - qrotation.qw = 0.25 / temp; - qrotation.qx = (rotMatrix[7] - rotMatrix[5]) * temp; - qrotation.qy = (rotMatrix[2] - rotMatrix[6]) * temp; - qrotation.qz = (rotMatrix[3] - rotMatrix[1]) * temp; - } - else - { - if (rotMatrix[0] > rotMatrix[4] && rotMatrix[0] > rotMatrix[8]) - { - double temp = 2 * std::sqrt(rotMatrix[0] - rotMatrix[4] - rotMatrix[8] + 1.0); - qrotation.qx = 0.25 * temp; - qrotation.qw = (rotMatrix[7] - rotMatrix[5]) * temp; - qrotation.qy = (rotMatrix[2] - rotMatrix[6]) * temp; - qrotation.qz = (rotMatrix[3] - rotMatrix[1]) * temp; - } - else if (rotMatrix[0] > rotMatrix[8]) - { - double temp = 2 * std::sqrt(rotMatrix[4] - rotMatrix[0] - rotMatrix[8] + 1.0); - qrotation.qy = 0.25 * temp; - qrotation.qx = (rotMatrix[7] - rotMatrix[5]) * temp; - qrotation.qw = (rotMatrix[2] - rotMatrix[6]) * temp; - qrotation.qz = (rotMatrix[3] - rotMatrix[1]) * temp; - } - else - { - double temp = 2 * std::sqrt(rotMatrix[8] - rotMatrix[4] - rotMatrix[0] + 1.0); - qrotation.qz = 0.25 * temp; - qrotation.qx = (rotMatrix[7] - rotMatrix[5]) * temp; - qrotation.qy = (rotMatrix[2] - rotMatrix[6]) * temp; - qrotation.qw = (rotMatrix[3] - rotMatrix[1]) * temp; - } - } - return qrotation; -} - -} // namespace cvcore diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/core/cvcore/Tensor.cpp b/isaac_ros_bi3d/gxf/bi3d/cvcore/src/core/cvcore/Tensor.cpp deleted file mode 100644 index 83ede60..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/core/cvcore/Tensor.cpp +++ /dev/null @@ -1,270 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2019-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -#include "cv/core/Tensor.h" - -#include - -#include -#include -#include -#include -#include - -namespace cvcore { - -TensorBase::TensorBase() - : m_data(nullptr) - , m_dimCount(0) - , m_type(U8) - , m_isOwning(false) - , m_isCPU(true) -{ - for (int i = 0; i < kMaxDimCount; ++i) - { - m_dimData[i] = {0, 0}; - } -} - -TensorBase::TensorBase(ChannelType type, const DimData *dimData, int dimCount, void *dataPtr, bool isCPU) - : TensorBase() -{ - assert(dimCount >= kMinDimCount && dimCount <= kMaxDimCount); - - m_isOwning = false; - m_isCPU = isCPU; - - m_type = type; - m_dimCount = dimCount; - for (int i = 0; i < dimCount; ++i) - { - m_dimData[i] = dimData[i]; - } - - m_data = dataPtr; -} - -TensorBase::TensorBase(ChannelType type, std::initializer_list dimData, void *dataPtr, bool isCPU) - : TensorBase(type, dimData.begin(), dimData.size(), dataPtr, isCPU) -{ -} - -TensorBase::TensorBase(ChannelType type, const DimData *dimData, int dimCount, bool isCPU) - : TensorBase(type, dimData, dimCount, nullptr, isCPU) -{ - m_isOwning = true; - - // compute tensor memory block size - const std::size_t tensorSize = getDataSize(); - - // allocate - if (isCPU) - { - m_data = std::malloc(tensorSize); - } - else - { - if (cudaMalloc(&m_data, tensorSize) != 0) - { - throw std::runtime_error("CUDA alloc() failed!"); - } - } -} - -TensorBase::TensorBase(ChannelType type, std::initializer_list dimData, bool isCPU) - : TensorBase(type, dimData.begin(), dimData.size(), isCPU) -{ -} - -TensorBase::~TensorBase() -{ - if (m_isOwning) - { - if (m_isCPU) - { - std::free(m_data); - } - else - { - cudaFree(m_data); - } - } -} - -TensorBase::TensorBase(TensorBase &&t) - : TensorBase() -{ - *this = std::move(t); -} - -TensorBase &TensorBase::operator=(TensorBase &&t) -{ - using std::swap; - - swap(m_data, t.m_data); - swap(m_dimCount, t.m_dimCount); - swap(m_type, t.m_type); - swap(m_isOwning, t.m_isOwning); - swap(m_isCPU, t.m_isCPU); - - for (int i = 0; i < kMaxDimCount; ++i) - { - swap(m_dimData[i], t.m_dimData[i]); - } - - return *this; -} - -int TensorBase::getDimCount() const -{ - return m_dimCount; -} - -std::size_t TensorBase::getSize(int dimIdx) const -{ - assert(dimIdx >= 0 && dimIdx < m_dimCount); - return m_dimData[dimIdx].size; -} - -std::size_t TensorBase::getStride(int dimIdx) const -{ - assert(dimIdx >= 0 && dimIdx < m_dimCount); - return m_dimData[dimIdx].stride; -} - -ChannelType TensorBase::getType() const -{ - return m_type; -} - -void *TensorBase::getData() const -{ - return m_data; -} - -std::size_t TensorBase::getDataSize() const -{ - std::size_t tensorSize = m_dimData[0].size * m_dimData[0].stride; - for (int i = 1; i < m_dimCount; ++i) - { - tensorSize = std::max(tensorSize, m_dimData[i].size * m_dimData[i].stride); - } - tensorSize *= GetChannelSize(m_type); - return tensorSize; -} - -bool TensorBase::isCPU() const -{ - return m_isCPU; -} - -bool TensorBase::isOwning() const -{ - return m_isOwning; -} - -std::string GetTensorLayoutAsString(TensorLayout TL) -{ - switch (TL) - { - case TensorLayout::CL: - return "CL"; - case TensorLayout::LC: - return "LC"; - case TensorLayout::HWC: - return "HWC"; - case TensorLayout::CHW: - return "CHW"; - case TensorLayout::DHWC: - return "DHWC"; - case TensorLayout::DCHW: - return "DCHW"; - case TensorLayout::CDHW: - return "CDHW"; - default: - throw std::runtime_error("Invalid TensorLayout"); - } -} - -std::string GetChannelCountAsString(ChannelCount CC) -{ - switch (CC) - { - case ChannelCount::C1: - return "C1"; - case ChannelCount::C2: - return "C2"; - case ChannelCount::C3: - return "C3"; - case ChannelCount::C4: - return "C4"; - case ChannelCount::CX: - return "CX"; - default: - throw std::runtime_error("Invalid ChannelCount"); - } -} - -std::string GetChannelTypeAsString(ChannelType CT) -{ - switch (CT) - { - case ChannelType::U8: - return "U8"; - case ChannelType::U16: - return "U16"; - case ChannelType::S8: - return "S8"; - case ChannelType::S16: - return "S16"; - case ChannelType::F16: - return "F16"; - case ChannelType::F32: - return "F32"; - case ChannelType::F64: - return "F64"; - default: - throw std::runtime_error("Invalid ChannelType"); - } -} - -std::size_t GetChannelSize(ChannelType CT) -{ - switch (CT) - { - case U8: - case S8: - return 1; - case F16: - case U16: - case S16: - return 2; - case F32: - return 4; - case F64: - return 8; - default: - throw std::runtime_error("Invalid ChannelType"); - } -} - -std::string GetMemoryTypeAsString(bool isCPU) -{ - return isCPU? "CPU" : "GPU"; -} - -} // namespace cvcore diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/core/utility/CVError.cpp b/isaac_ros_bi3d/gxf/bi3d/cvcore/src/core/utility/CVError.cpp deleted file mode 100644 index a6e62c7..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/core/utility/CVError.cpp +++ /dev/null @@ -1,123 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -#include "cv/core/CVError.h" - -#include -#include -#include - -#ifndef __cpp_lib_to_underlying -// Using a C++23 feature by hacking std -namespace std -{ - template - constexpr underlying_type_t to_underlying(Enum e) noexcept - { - return static_cast>(e); - } -}; -#endif // __cpp_lib_to_underlying - -namespace cvcore { - -namespace detail -{ - struct CoreErrorCategory : std::error_category - { - virtual const char * name() const noexcept override final - { - return "cvcore-error"; - } - - virtual std::string message(int value) const override final - { - std::string result; - - switch(value) - { - case std::to_underlying(ErrorCode::SUCCESS): - result = "(SUCCESS) No errors detected"; - break; - case std::to_underlying(ErrorCode::NOT_READY): - result = "(NOT_READY) The execution of the requested " - "operation is not to return"; - break; - case std::to_underlying(ErrorCode::NOT_IMPLEMENTED): - result = "(NOT_IMPLEMENTED) The requested operation is not " - "implemented"; - break; - case std::to_underlying(ErrorCode::INVALID_ARGUMENT): - result = "(INVALID_ARGUMENT) The argument provided to the " - "operation is not currently supported"; - break; - case std::to_underlying(ErrorCode::INVALID_IMAGE_FORMAT): - result = "(INVALID_IMAGE_FORMAT) The requested image format " - "is not supported by the operation"; - break; - case std::to_underlying(ErrorCode::INVALID_STORAGE_TYPE): - result = "(INVALID_STORAGE_TYPE) The requested storage type " - "is not supported by the operation"; - break; - case std::to_underlying(ErrorCode::INVALID_ENGINE_TYPE): - result = "(INVALID_ENGINE_TYPE) The requested engine type " - "is not supported by the operation"; - break; - case std::to_underlying(ErrorCode::INVALID_OPERATION): - result = "(INVALID_OPERATION) The requested operation is " - "not supported"; - break; - case std::to_underlying(ErrorCode::DETECTED_NAN_IN_RESULT): - result = "(DETECTED_NAN_IN_RESULT) NaN was detected in the " - "return value of the operation"; - break; - case std::to_underlying(ErrorCode::OUT_OF_MEMORY): - result = "(OUT_OF_MEMORY) The device has run out of memory"; - break; - case std::to_underlying(ErrorCode::DEVICE_ERROR): - result = "(DEVICE_ERROR) A device level error has been " - "encountered"; - break; - case std::to_underlying(ErrorCode::SYSTEM_ERROR): - result = "(SYSTEM_ERROR) A system level error has been " - "encountered"; - break; - default: - result = "(Unrecognized Condition) Value " + std::to_string(value) + - " does not map to known error code literal " + - " defined by cvcore::ErrorCode"; - break; - } - - return result; - } - }; -} // namespace detail - -const detail::CoreErrorCategory errorCategory{}; - -std::error_condition make_error_condition(ErrorCode ec) noexcept -{ - return {std::to_underlying(ec), errorCategory}; -} - -std::error_code make_error_code(ErrorCode ec) noexcept -{ - return {std::to_underlying(ec), errorCategory}; -} - -} // namespace cvcore diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/core/utility/Instrumentation.cpp b/isaac_ros_bi3d/gxf/bi3d/cvcore/src/core/utility/Instrumentation.cpp deleted file mode 100644 index 583b646..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/core/utility/Instrumentation.cpp +++ /dev/null @@ -1,95 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -#include "cv/core/Instrumentation.h" - -#ifdef NVBENCH_ENABLE -#include -#include -#include -#endif - -namespace cvcore { namespace profiler { - -#ifdef NVBENCH_ENABLE -nv::bench::JsonHelper mapProfilerJsonOutputTypeToNvbenchType(ProfilerJsonOutputType jsonType) -{ - nv::bench::JsonHelper nvbenchJsonOutputType = nv::bench::JsonHelper::JSON_OFF; - if (jsonType == ProfilerJsonOutputType::JSON_OFF) - { - nvbenchJsonOutputType = nv::bench::JsonHelper::JSON_OFF; - } - else if (jsonType == ProfilerJsonOutputType::JSON_SEPARATE) - { - nvbenchJsonOutputType = nv::bench::JsonHelper::JSON_SEPARATE; - } - else if (jsonType == ProfilerJsonOutputType::JSON_AGGREGATE) - { - nvbenchJsonOutputType = nv::bench::JsonHelper::JSON_AGGREGATE; - } - return nvbenchJsonOutputType; -} -#endif - -void flush(const std::string& filename, ProfilerJsonOutputType jsonType) -{ -#ifdef NVBENCH_ENABLE - nv::bench::JsonHelper nvbenchJsonOutputType = mapProfilerJsonOutputTypeToNvbenchType(jsonType); - if (!filename.empty()) - { - nv::bench::Pool::instance().flushToFile(filename.c_str(), -1, INT_MAX, nvbenchJsonOutputType); - } - else - { - nv::bench::Pool::instance().flush(std::clog, -1, INT_MAX, nvbenchJsonOutputType); - } -#else - return; -#endif - -} - -void flush(std::ostream& output, ProfilerJsonOutputType jsonType) -{ -#ifdef NVBENCH_ENABLE - nv::bench::JsonHelper nvbenchJsonOutputType = mapProfilerJsonOutputTypeToNvbenchType(jsonType); - nv::bench::Pool::instance().flush(output, -1, INT_MAX, nvbenchJsonOutputType); -#else - return; -#endif -} - -void flush(ProfilerJsonOutputType jsonType) -{ -#ifdef NVBENCH_ENABLE - nv::bench::JsonHelper nvbenchJsonOutputType = mapProfilerJsonOutputTypeToNvbenchType(jsonType); - nv::bench::Pool::instance().flush(std::clog, -1, INT_MAX, nvbenchJsonOutputType); -#else - return; -#endif -} - -void clear() -{ -#ifdef NVBENCH_ENABLE - nv::bench::Pool::instance().clear(); -#else - return; -#endif -} - -}} diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/core/utility/Memory.cpp b/isaac_ros_bi3d/gxf/bi3d/cvcore/src/core/utility/Memory.cpp deleted file mode 100644 index e75a614..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/core/utility/Memory.cpp +++ /dev/null @@ -1,124 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2020-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -#include "cv/core/Memory.h" - -#include - -#include -#include -#include - -namespace cvcore { - -namespace { - -// Copy 2D CPU pitch linear tensors -void Memcpy2DCPU(void *dst, size_t dstPitch, const void *src, size_t srcPitch, size_t widthInBytes, size_t height) -{ - uint8_t *dstPt = reinterpret_cast(dst); - const uint8_t *srcPt = reinterpret_cast(src); - for (size_t i = 0; i < height; i++) - { - memcpy(dstPt, srcPt, widthInBytes); - dstPt += dstPitch; - srcPt += srcPitch; - } -} - -} // anonymous namespace - -void TensorBaseCopy(TensorBase &dst, const TensorBase &src, cudaStream_t stream) -{ - if (dst.getDataSize() != src.getDataSize()) - { - throw std::runtime_error("Tensor stride mismatch!"); - } - assert(dst.getDimCount() == src.getDimCount()); - int dimCount = src.getDimCount(); - for (int i = 0; i < dimCount - 1; i++) - { - if (src.getStride(i) != src.getStride(i + 1) * src.getSize(i + 1) || - dst.getStride(i) != dst.getStride(i + 1) * dst.getSize(i + 1)) - { - throw std::runtime_error("Tensor is not contiguous in memory!"); - } - } - if (dst.isCPU() && src.isCPU()) - { - memcpy(dst.getData(), src.getData(), src.getDataSize()); - return; - } - cudaError_t error; - if (!dst.isCPU() && src.isCPU()) - { - error = cudaMemcpyAsync(dst.getData(), src.getData(), src.getDataSize(), cudaMemcpyHostToDevice, stream); - } - else if (dst.isCPU() && !src.isCPU()) - { - error = cudaMemcpyAsync(dst.getData(), src.getData(), src.getDataSize(), cudaMemcpyDeviceToHost, stream); - } - else - { - error = cudaMemcpyAsync(dst.getData(), src.getData(), src.getDataSize(), cudaMemcpyDeviceToDevice, stream); - } - if (error != cudaSuccess) - { - throw std::runtime_error("CUDA memcpy failed!"); - } -} - -void TensorBaseCopy2D(TensorBase &dst, const TensorBase &src, int dstPitch, int srcPitch, int widthInBytes, int height, - cudaStream_t stream) -{ - assert(dst.getDimCount() == src.getDimCount()); - int dimCount = src.getDimCount(); - for (int i = 0; i < dimCount; i++) - { - if (dst.getSize(i) != src.getSize(i)) - { - throw std::runtime_error("Tensor size mismatch!"); - } - } - if (dst.isCPU() && src.isCPU()) - { - Memcpy2DCPU(dst.getData(), dstPitch, src.getData(), srcPitch, widthInBytes, height); - return; - } - cudaError_t error; - if (!dst.isCPU() && src.isCPU()) - { - error = cudaMemcpy2DAsync(dst.getData(), dstPitch, src.getData(), srcPitch, widthInBytes, height, - cudaMemcpyHostToDevice, stream); - } - else if (dst.isCPU() && !src.isCPU()) - { - error = cudaMemcpy2DAsync(dst.getData(), dstPitch, src.getData(), srcPitch, widthInBytes, height, - cudaMemcpyDeviceToHost, stream); - } - else - { - error = cudaMemcpy2DAsync(dst.getData(), dstPitch, src.getData(), srcPitch, widthInBytes, height, - cudaMemcpyDeviceToDevice, stream); - } - if (error != cudaSuccess) - { - throw std::runtime_error("CUDA memcpy failed!"); - } -} - -} // namespace cvcore diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/core/utility/ProfileUtils.cpp b/isaac_ros_bi3d/gxf/bi3d/cvcore/src/core/utility/ProfileUtils.cpp deleted file mode 100644 index 53c5811..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/core/utility/ProfileUtils.cpp +++ /dev/null @@ -1,128 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2020-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -#include - -#include - -#include -#include -#include - -#if defined(_MSC_VER) || defined(__WIN32) -# include -# include -# include -#endif - -using json = nlohmann::json; - -namespace cvcore { - -namespace { - -#if defined(_MSC_VER) || defined(__WIN32) -std::string GetCPUName() -{ - // https://docs.microsoft.com/en-us/previous-versions/visualstudio/visual-studio-2008/hskdteyh(v=vs.90)?redirectedfrom=MSDN - char CPUBrandString[0x40]; - int CPUInfo[4] = {-1}; - - // Calling __cpuid with 0x80000000 as the InfoType argument - // gets the number of valid extended IDs. - __cpuid(CPUInfo, 0x80000000); - unsigned i, nExIds = CPUInfo[0]; - memset(CPUBrandString, 0, sizeof(CPUBrandString)); - - // Get the information associated with each extended ID. - for (i=0x80000000; i<=nExIds; ++i) - { - __cpuid(CPUInfo, i); - - // Interpret CPU brand string and cache information. - if (i == 0x80000002) - memcpy(CPUBrandString, CPUInfo, sizeof(CPUInfo)); - else if (i == 0x80000003) - memcpy(CPUBrandString + 16, CPUInfo, sizeof(CPUInfo)); - else if (i == 0x80000004) - memcpy(CPUBrandString + 32, CPUInfo, sizeof(CPUInfo)); - } - return CPUBrandString; -} -#else -std::string GetCPUName() -{ - // TODO: this will only work on linux platform - std::ifstream cpuInfo("/proc/cpuinfo"); - if (!cpuInfo.good()) - { - throw std::runtime_error("unable to retrieve cpu info"); - } - std::string line; - while (std::getline(cpuInfo, line)) - { - int delimiterPos = line.find(':'); - if (delimiterPos != std::string::npos) - { - std::string key = line.substr(0, delimiterPos); - if (key.find("model name") != std::string::npos) - { - std::string info = line.substr(delimiterPos + 1); - info.erase(0, info.find_first_not_of(' ')); - return info; - } - } - } - return "CPU"; // default name if no cpu model name retrieved -} -#endif - -std::string GetGPUName() -{ - int deviceId; - cudaGetDevice(&deviceId); - cudaDeviceProp prop; - cudaError_t error = cudaGetDeviceProperties(&prop, deviceId); - if (error != 0) - { - throw std::runtime_error("unable to retrieve cuda device info"); - } - return std::string(prop.name); -} - -} // anonymous namespace - -void ExportToJson(const std::string outputPath, const std::string taskName, float tMin, float tMax, float tAvg, - bool isCPU, int iterations = 100) -{ - std::ifstream in(outputPath); - json jsonHandler; - if (in.good()) - { - in >> jsonHandler; - } - in.close(); - - const std::string platform = isCPU ? "CPU: " + GetCPUName() : "GPU: " + GetGPUName(); - jsonHandler[platform][taskName] = {{"iter", iterations}, {"min", tMin}, {"max", tMax}, {"avg", tAvg}}; - - std::ofstream out(outputPath); - out << std::setw(4) << jsonHandler << std::endl; - out.close(); -} - -} // namespace cvcore diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/inferencer/tensorrt/TensorRTInferencer.cpp b/isaac_ros_bi3d/gxf/bi3d/cvcore/src/inferencer/tensorrt/TensorRTInferencer.cpp deleted file mode 100644 index db5a0d0..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/inferencer/tensorrt/TensorRTInferencer.cpp +++ /dev/null @@ -1,275 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 -#include "TensorRTInferencer.h" -#include -#include -#include "TensorRTUtils.h" - -#include "cv/inferencer/Errors.h" -#include "cv/inferencer/IInferenceBackend.h" -#include "cv/inferencer/Inferencer.h" - -namespace cvcore { namespace inferencer { - -namespace { -size_t getDataSize(const std::vector &shape, cvcore::ChannelType dataType) -{ - size_t layerShape = 1; - for (size_t k = 0; k < shape.size(); k++) - layerShape *= shape[k] <= 0 ? 1 : shape[k]; - - return layerShape * GetChannelSize(dataType); -} -} // namespace - -std::error_code TensorRTInferencer::getLayerInfo(LayerInfo &layer, std::string layerName) -{ - layer.name = layerName; - layer.index = m_inferenceEngine->getBindingIndex(layerName.c_str()); - auto dim = m_inferenceEngine->getBindingDimensions(layer.index); - nvinfer1::TensorFormat tensorFormat = m_inferenceEngine->getBindingFormat(layer.index); - - std::error_code err; - err = getCVCoreChannelLayoutFromTensorRT(layer.layout, tensorFormat); - if (err != cvcore::make_error_code(ErrorCode::SUCCESS)) - { - return ErrorCode::INVALID_ARGUMENT; - } - - for (size_t cnt = 0; cnt < dim.nbDims; cnt++) - { - layer.shape.push_back(dim.d[cnt]); - } - - err = getCVCoreChannelTypeFromTensorRT(layer.dataType, m_inferenceEngine->getBindingDataType(layer.index)); - layer.layerSize = getDataSize(layer.shape, layer.dataType); - if (err != cvcore::make_error_code(ErrorCode::SUCCESS)) - { - return ErrorCode::INVALID_ARGUMENT; - } - - return ErrorCode::SUCCESS; -} - -std::error_code TensorRTInferencer::ParseTRTModel() -{ - - m_modelInfo.modelName = m_inferenceEngine->getName(); - m_modelInfo.modelVersion = ""; - m_modelInfo.maxBatchSize = m_maxBatchSize; - std::error_code err; - for (size_t i = 0; i < m_inputLayers.size(); i++) - { - LayerInfo layer; - err = getLayerInfo(layer, m_inputLayers[i]); - if (err != cvcore::make_error_code(cvcore::ErrorCode::SUCCESS)) - { - return err; - } - m_modelInfo.inputLayers[layer.name] = layer; - } - for (size_t i = 0; i < m_outputLayers.size(); i++) - { - LayerInfo layer; - err = getLayerInfo(layer, m_outputLayers[i]); - if (err != cvcore::make_error_code(cvcore::ErrorCode::SUCCESS)) - { - return err; - } - m_modelInfo.outputLayers[layer.name] = layer; - } - - return ErrorCode::SUCCESS; -} - -TensorRTInferencer::TensorRTInferencer(const TensorRTInferenceParams ¶ms) - : m_logger(new TRTLogger()) - , m_maxBatchSize(params.maxBatchSize) - , m_inputLayers(params.inputLayerNames) - , m_outputLayers(params.outputLayerNames) - , m_cudaStream(0) - , m_inferenceEngine(nullptr) -{ - - if (params.inferType == TRTInferenceType::TRT_ENGINE) - { - std::ifstream trtModelFStream(params.engineFilePath, std::ios::binary); - std::unique_ptr trtModelContent; - size_t trtModelContentSize = 0; - - if (!trtModelFStream.good()) - { - throw ErrorCode::INVALID_ARGUMENT; - } - else - { - trtModelFStream.seekg(0, trtModelFStream.end); - trtModelContentSize = trtModelFStream.tellg(); - trtModelFStream.seekg(0, trtModelFStream.beg); - trtModelContent.reset(new char[trtModelContentSize]); - trtModelFStream.read(trtModelContent.get(), trtModelContentSize); - trtModelFStream.close(); - } - - m_inferenceRuntime.reset(nvinfer1::createInferRuntime(*(m_logger.get()))); - if (params.dlaID != -1 && params.dlaID < m_inferenceRuntime->getNbDLACores()) - { - m_inferenceRuntime->setDLACore(params.dlaID); - } - m_inferenceEngine = m_inferenceRuntime->deserializeCudaEngine(trtModelContent.get(), trtModelContentSize); - m_ownedInferenceEngine.reset(m_inferenceEngine); - m_inferenceContext.reset(m_inferenceEngine->createExecutionContext()); - m_inferenceContext->setOptimizationProfileAsync(0, m_cudaStream); - } - else - { - if (params.engine == nullptr) - { - throw ErrorCode::INVALID_ARGUMENT; - } - m_inferenceEngine = params.engine; - m_inferenceContext.reset(m_inferenceEngine->createExecutionContext()); - } - - if (m_inferenceEngine == nullptr || m_inferenceContext == nullptr) - { - throw ErrorCode::INVALID_ARGUMENT; - } - - m_hasImplicitBatch = m_inferenceEngine->hasImplicitBatchDimension(); - m_bindingsCount = m_inferenceEngine->getNbBindings(); - if (!m_hasImplicitBatch) - { - for (size_t i = 0; i < m_bindingsCount; i++) - { - if (m_inferenceEngine->bindingIsInput(i)) - { - nvinfer1::Dims dims_i(m_inferenceEngine->getBindingDimensions(i)); - nvinfer1::Dims4 inputDims{1, dims_i.d[1], dims_i.d[2], dims_i.d[3]}; - m_inferenceContext->setBindingDimensions(i, inputDims); - } - } - } - std::error_code err; - err = ParseTRTModel(); - if (err != cvcore::make_error_code(ErrorCode::SUCCESS)) - { - throw err; - } - m_buffers.resize(m_bindingsCount); -} - -// Set input layer tensor -std::error_code TensorRTInferencer::setInput(const cvcore::TensorBase &trtInputBuffer, std::string inputLayerName) -{ - if (m_modelInfo.inputLayers.find(inputLayerName) == m_modelInfo.inputLayers.end()) - { - return ErrorCode::INVALID_ARGUMENT; - } - LayerInfo layer = m_modelInfo.inputLayers[inputLayerName]; - m_buffers[layer.index] = trtInputBuffer.getData(); - return ErrorCode::SUCCESS; -} - -// Sets output layer tensor -std::error_code TensorRTInferencer::setOutput(cvcore::TensorBase &trtOutputBuffer, std::string outputLayerName) -{ - if (m_modelInfo.outputLayers.find(outputLayerName) == m_modelInfo.outputLayers.end()) - { - return ErrorCode::INVALID_ARGUMENT; - } - LayerInfo layer = m_modelInfo.outputLayers[outputLayerName]; - m_buffers[layer.index] = trtOutputBuffer.getData(); - return ErrorCode::SUCCESS; -} - -// Get the model metadata parsed based on the model file -// This would be done in initialize call itself. User can access the modelMetaData created using this API. -ModelMetaData TensorRTInferencer::getModelMetaData() const -{ - return m_modelInfo; -} - -std::error_code TensorRTInferencer::infer(size_t batchSize) -{ - bool err = true; - if (!m_hasImplicitBatch) - { - size_t bindingsCount = m_inferenceEngine->getNbBindings(); - for (size_t i = 0; i < bindingsCount; i++) - { - if (m_inferenceEngine->bindingIsInput(i)) - { - nvinfer1::Dims dims_i(m_inferenceEngine->getBindingDimensions(i)); - nvinfer1::Dims4 inputDims{static_cast(batchSize), dims_i.d[1], dims_i.d[2], dims_i.d[3]}; - m_inferenceContext->setBindingDimensions(i, inputDims); - } - } - err = m_inferenceContext->enqueueV2(&m_buffers[0], m_cudaStream, nullptr); - } - else - { - err = m_inferenceContext->enqueue(m_maxBatchSize, &m_buffers[0], m_cudaStream, nullptr); - } - if (!err) - { - return InferencerErrorCode::TENSORRT_INFERENCE_ERROR; - } - return ErrorCode::SUCCESS; -} - -// Applicable only for Native TRT -std::error_code TensorRTInferencer::setCudaStream(cudaStream_t cudaStream) // Only in TRT -{ - m_cudaStream = cudaStream; - return ErrorCode::SUCCESS; -} - -std::error_code TensorRTInferencer::unregister(std::string layerName) -{ - size_t index; - if (m_modelInfo.outputLayers.find(layerName) != m_modelInfo.outputLayers.end()) - { - index = m_modelInfo.outputLayers[layerName].index; - } - else if (m_modelInfo.inputLayers.find(layerName) != m_modelInfo.inputLayers.end()) - { - index = m_modelInfo.inputLayers[layerName].index; - } - else - { - return ErrorCode::INVALID_ARGUMENT; - } - m_buffers[index] = nullptr; - return ErrorCode::SUCCESS; -} - -std::error_code TensorRTInferencer::unregister() -{ - for (size_t i = 0; i < m_buffers.size(); i++) - { - m_buffers[i] = nullptr; - } - return ErrorCode::SUCCESS; -} - -TensorRTInferencer::~TensorRTInferencer() -{ - m_buffers.clear(); -} - -}} // namespace cvcore::inferencer diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/inferencer/tensorrt/TensorRTUtils.cpp b/isaac_ros_bi3d/gxf/bi3d/cvcore/src/inferencer/tensorrt/TensorRTUtils.cpp deleted file mode 100644 index dfc8919..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/inferencer/tensorrt/TensorRTUtils.cpp +++ /dev/null @@ -1,65 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -#include "TensorRTUtils.h" -#include - -namespace cvcore { namespace inferencer { - -std::error_code getCVCoreChannelTypeFromTensorRT(cvcore::ChannelType &channelType, nvinfer1::DataType dtype) -{ - if (dtype == nvinfer1::DataType::kINT8) - { - channelType = cvcore::ChannelType::U8; - } - else if (dtype == nvinfer1::DataType::kHALF) - { - channelType = cvcore::ChannelType::F16; - } - else if (dtype == nvinfer1::DataType::kFLOAT) - { - channelType = cvcore::ChannelType::F32; - } - else - { - return ErrorCode::INVALID_OPERATION; - } - - return ErrorCode::SUCCESS; -} - -std::error_code getCVCoreChannelLayoutFromTensorRT(cvcore::TensorLayout &channelLayout, - nvinfer1::TensorFormat tensorFormat) -{ - if (tensorFormat == nvinfer1::TensorFormat::kLINEAR || - tensorFormat == nvinfer1::TensorFormat::kCHW32) - { - channelLayout = cvcore::TensorLayout::NCHW; - } - else if (tensorFormat == nvinfer1::TensorFormat::kHWC) - { - channelLayout = cvcore::TensorLayout::HWC; - } - else - { - return ErrorCode::INVALID_OPERATION; - } - - return ErrorCode::SUCCESS; -} - -}} // namespace cvcore::inferencer diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/inferencer/triton/TritonUtils.cpp b/isaac_ros_bi3d/gxf/bi3d/cvcore/src/inferencer/triton/TritonUtils.cpp deleted file mode 100644 index 47b3653..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/inferencer/triton/TritonUtils.cpp +++ /dev/null @@ -1,84 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 -#ifdef ENABLE_TRITON -#include "TritonUtils.h" -#include - -namespace cvcore { namespace inferencer { - -bool getCVCoreChannelType(cvcore::ChannelType &channelType, std::string dtype) -{ - if (dtype.compare("UINT8") == 0) - { - channelType = cvcore::ChannelType::U8; - } - else if (dtype.compare("UINT16") == 0) - { - channelType = cvcore::ChannelType::U16; - } - else if (dtype.compare("FP16") == 0) - { - channelType = cvcore::ChannelType::F16; - } - else if (dtype.compare("FP32") == 0) - { - channelType = cvcore::ChannelType::F32; - } - else if (dtype.compare("FP64") == 0) - { - channelType = cvcore::ChannelType::F64; - } - else - { - return false; - } - - return true; -} - -bool getTritonChannelType(std::string &dtype, cvcore::ChannelType channelType) -{ - if (channelType == cvcore::ChannelType::U8) - { - dtype = "UINT8"; - } - else if (channelType == cvcore::ChannelType::U16) - { - dtype = "UINT16"; - } - else if (channelType == cvcore::ChannelType::F16) - { - dtype = "FP16"; - } - else if (channelType == cvcore::ChannelType::F32) - { - dtype = "FP32"; - } - else if (channelType == cvcore::ChannelType::F64) - { - dtype = "FP64"; - } - else - { - return false; - } - - return true; -} - -}} // namespace cvcore::inferencer -#endif diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/ArithmeticOperations.cpp b/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/ArithmeticOperations.cpp deleted file mode 100644 index 85d5e2f..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/ArithmeticOperations.cpp +++ /dev/null @@ -1,329 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2020-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -#include "NppUtils.h" - -#include "cv/tensor_ops/ImageUtils.h" - -#include "cv/core/Memory.h" - -#include -#include - -#include -#include -#include - -namespace cvcore { namespace tensor_ops { - -namespace { - -static void NormalizeTensorC3F32Inplace(Tensor &src, const float scale[3], const float offset[3], - NppStreamContext streamContext) -{ - const int srcW = src.getWidth(); - const int srcH = src.getHeight(); - const NppiSize srcSize = {srcW, srcH}; - - const Npp32f offsets[3] = {static_cast(offset[0]), static_cast(offset[1]), - static_cast(offset[2])}; - NppStatus status = - nppiAddC_32f_C3IR_Ctx(offsets, static_cast(src.getData()), - src.getStride(TensorDimension::HEIGHT) * sizeof(Npp32f), srcSize, streamContext); - assert(status == NPP_SUCCESS); - - const Npp32f scales[3] = {static_cast(scale[0]), static_cast(scale[1]), - static_cast(scale[2])}; - status = nppiMulC_32f_C3IR_Ctx(scales, static_cast(src.getData()), - src.getStride(TensorDimension::HEIGHT) * sizeof(Npp32f), srcSize, streamContext); - assert(status == NPP_SUCCESS); -} - -template -static void NormalizeTensorC1F32Inplace(Tensor &src, const float scale, const float offset, - NppStreamContext streamContext) -{ - const int srcW = src.getWidth(); - const int srcH = src.getHeight(); - const NppiSize srcSize = {srcW, srcH}; - - NppStatus status = - nppiAddC_32f_C1IR_Ctx(static_cast(offset), static_cast(src.getData()), - src.getStride(TensorDimension::HEIGHT) * sizeof(Npp32f), srcSize, streamContext); - assert(status == NPP_SUCCESS); - - status = nppiMulC_32f_C1IR_Ctx(static_cast(scale), static_cast(src.getData()), - src.getStride(TensorDimension::HEIGHT) * sizeof(Npp32f), srcSize, streamContext); - assert(status == NPP_SUCCESS); -} - -template -void NormalizeC1U8Impl(Tensor &dst, const Tensor &src, const float scale, const float offset, - cudaStream_t stream) -{ - // src and dst must be GPU tensors - assert(!src.isCPU() && !dst.isCPU()); - assert((src.getWidth() == dst.getWidth()) && (src.getHeight() == dst.getHeight())); - - NppStreamContext streamContext = GetNppStreamContext(stream); - - NppStatus status = nppiConvert_8u32f_C1R_Ctx( - static_cast(src.getData()), src.getStride(TensorDimension::HEIGHT) * sizeof(Npp8u), - static_cast(dst.getData()), dst.getStride(TensorDimension::HEIGHT) * sizeof(Npp32f), - {int(src.getWidth()), int(src.getHeight())}, streamContext); - assert(status == NPP_SUCCESS); - - NormalizeTensorC1F32Inplace(dst, scale, offset, streamContext); -} - -template -void NormalizeC1U16Impl(Tensor &dst, const Tensor &src, const float scale, const float offset, - cudaStream_t stream) -{ - // src and dst must be GPU tensors - assert(!src.isCPU() && !dst.isCPU()); - assert((src.getWidth() == dst.getWidth()) && (src.getHeight() == dst.getHeight())); - - NppStreamContext streamContext = GetNppStreamContext(stream); - - NppStatus status = nppiConvert_16u32f_C1R_Ctx( - static_cast(src.getData()), src.getStride(TensorDimension::HEIGHT) * sizeof(Npp16u), - static_cast(dst.getData()), dst.getStride(TensorDimension::HEIGHT) * sizeof(Npp32f), - {int(src.getWidth()), int(src.getHeight())}, streamContext); - assert(status == NPP_SUCCESS); - - NormalizeTensorC1F32Inplace(dst, scale, offset, streamContext); -} - -template -void NormalizeC1F32Impl(Tensor &dst, const Tensor &src, const float scale, const float offset, - cudaStream_t stream) -{ - // src and dst must be GPU tensors - assert(!src.isCPU() && !dst.isCPU()); - assert((src.getWidth() == dst.getWidth()) && (src.getHeight() == dst.getHeight())); - - Copy(dst, src, stream); - NormalizeTensorC1F32Inplace(dst, scale, offset, GetNppStreamContext(stream)); -} - -template -void NormalizeC3Batch(Tensor &dst, Tensor &src, const float scale[3], - const float offset[3], cudaStream_t stream) -{ - // src and dst must be GPU tensors - assert(!src.isCPU() && !dst.isCPU()); - assert(src.getDepth() == dst.getDepth()); - - for (int i = 0; i < src.getDepth(); i++) - { - size_t shiftSrc = i * src.getStride(TensorDimension::DEPTH); - size_t shiftDst = i * dst.getStride(TensorDimension::DEPTH); - Tensor srcTmp(src.getWidth(), src.getHeight(), - src.getStride(TensorDimension::HEIGHT) * GetChannelSize(CT), - src.getData() + shiftSrc, false); - Tensor dstTmp(dst.getWidth(), dst.getHeight(), - dst.getStride(TensorDimension::HEIGHT) * GetChannelSize(F32), - dst.getData() + shiftDst, false); - Normalize(dstTmp, srcTmp, scale, offset, stream); - } -} - -template -void NormalizeC1Batch(Tensor &dst, Tensor &src, const float scale, const float offset, - cudaStream_t stream) -{ - // src and dst must be GPU tensors - assert(!src.isCPU() && !dst.isCPU()); - assert(src.getDepth() == dst.getDepth()); - - for (int i = 0; i < src.getDepth(); i++) - { - size_t shiftSrc = i * src.getStride(TensorDimension::DEPTH); - size_t shiftDst = i * dst.getStride(TensorDimension::DEPTH); - Tensor srcTmp(src.getWidth(), src.getHeight(), - src.getStride(TensorDimension::HEIGHT) * GetChannelSize(CT), - src.getData() + shiftSrc, false); - Tensor dstTmp(dst.getWidth(), dst.getHeight(), - dst.getStride(TensorDimension::HEIGHT) * GetChannelSize(F32), - dst.getData() + shiftDst, false); - Normalize(dstTmp, srcTmp, scale, offset, stream); - } -} - -template -void NormalizeC1Batch(Tensor &dst, Tensor &src, const float scale, const float offset, - cudaStream_t stream) -{ - // src and dst must be GPU tensors - assert(!src.isCPU() && !dst.isCPU()); - assert(src.getDepth() == dst.getDepth()); - - for (int i = 0; i < src.getDepth(); i++) - { - size_t shiftSrc = i * src.getStride(TensorDimension::DEPTH); - size_t shiftDst = i * dst.getStride(TensorDimension::DEPTH); - Tensor srcTmp(src.getWidth(), src.getHeight(), - src.getStride(TensorDimension::HEIGHT) * GetChannelSize(CT), - src.getData() + shiftSrc, false); - Tensor dstTmp(dst.getWidth(), dst.getHeight(), - dst.getStride(TensorDimension::HEIGHT) * GetChannelSize(F32), - dst.getData() + shiftDst, false); - Normalize(dstTmp, srcTmp, scale, offset, stream); - } -} - -} // anonymous namespace - -void Normalize(Tensor &dst, const Tensor &src, const float scale[3], const float offset[3], - cudaStream_t stream) -{ - // src and dst must be GPU tensors - assert(!src.isCPU() && !dst.isCPU()); - assert((src.getWidth() == dst.getWidth()) && (src.getHeight() == dst.getHeight())); - - NppStreamContext streamContext = GetNppStreamContext(stream); - - NppStatus status = nppiConvert_8u32f_C3R_Ctx( - static_cast(src.getData()), src.getStride(TensorDimension::HEIGHT) * sizeof(Npp8u), - static_cast(dst.getData()), dst.getStride(TensorDimension::HEIGHT) * sizeof(Npp32f), - {int(src.getWidth()), int(src.getHeight())}, streamContext); - assert(status == NPP_SUCCESS); - - NormalizeTensorC3F32Inplace(dst, scale, offset, streamContext); -} - -void Normalize(Tensor &dst, const Tensor &src, const float scale[3], const float offset[3], - cudaStream_t stream) -{ - NormalizeC3Batch(dst, const_cast &>(src), scale, offset, stream); -} - -void Normalize(Tensor &dst, const Tensor &src, const float scale[3], const float offset[3], - cudaStream_t stream) -{ - // src and dst must be GPU tensors - assert(!src.isCPU() && !dst.isCPU()); - assert((src.getWidth() == dst.getWidth()) && (src.getHeight() == dst.getHeight())); - - NppStreamContext streamContext = GetNppStreamContext(stream); - - NppStatus status = nppiConvert_16u32f_C3R_Ctx( - static_cast(src.getData()), src.getStride(TensorDimension::HEIGHT) * sizeof(Npp16u), - static_cast(dst.getData()), dst.getStride(TensorDimension::HEIGHT) * sizeof(Npp32f), - {int(src.getWidth()), int(src.getHeight())}, streamContext); - assert(status == NPP_SUCCESS); - - NormalizeTensorC3F32Inplace(dst, scale, offset, streamContext); -} - -void Normalize(Tensor &dst, const Tensor &src, const float scale[3], - const float offset[3], cudaStream_t stream) -{ - NormalizeC3Batch(dst, const_cast &>(src), scale, offset, stream); -} - -void Normalize(Tensor &dst, const Tensor &src, const float scale[3], const float offset[3], - cudaStream_t stream) -{ - // src and dst must be GPU tensors - assert(!src.isCPU() && !dst.isCPU()); - assert((src.getWidth() == dst.getWidth()) && (src.getHeight() == dst.getHeight())); - - Copy(dst, src, stream); - NormalizeTensorC3F32Inplace(dst, scale, offset, GetNppStreamContext(stream)); -} - -void Normalize(Tensor &dst, const Tensor &src, const float scale[3], - const float offset[3], cudaStream_t stream) -{ - NormalizeC3Batch(dst, const_cast &>(src), scale, offset, stream); -} - -void Normalize(Tensor &dst, const Tensor &src, const float scale, const float offset, - cudaStream_t stream) -{ - NormalizeC1U8Impl(dst, src, scale, offset, stream); -} - -void Normalize(Tensor &dst, const Tensor &src, const float scale, const float offset, - cudaStream_t stream) -{ - NormalizeC1Batch(dst, const_cast &>(src), scale, offset, stream); -} - -void Normalize(Tensor &dst, const Tensor &src, const float scale, const float offset, - cudaStream_t stream) -{ - NormalizeC1U16Impl(dst, src, scale, offset, stream); -} - -void Normalize(Tensor &dst, const Tensor &src, const float scale, const float offset, - cudaStream_t stream) -{ - NormalizeC1Batch(dst, const_cast &>(src), scale, offset, stream); -} - -void Normalize(Tensor &dst, const Tensor &src, const float scale, const float offset, - cudaStream_t stream) -{ - NormalizeC1F32Impl(dst, src, scale, offset, stream); -} - -void Normalize(Tensor &dst, const Tensor &src, const float scale, const float offset, - cudaStream_t stream) -{ - NormalizeC1Batch(dst, const_cast &>(src), scale, offset, stream); -} - -void Normalize(Tensor &dst, const Tensor &src, const float scale, const float offset, - cudaStream_t stream) -{ - NormalizeC1U8Impl(dst, src, scale, offset, stream); -} - -void Normalize(Tensor &dst, const Tensor &src, const float scale, const float offset, - cudaStream_t stream) -{ - NormalizeC1Batch(dst, const_cast &>(src), scale, offset, stream); -} - -void Normalize(Tensor &dst, const Tensor &src, const float scale, const float offset, - cudaStream_t stream) -{ - NormalizeC1U16Impl(dst, src, scale, offset, stream); -} - -void Normalize(Tensor &dst, const Tensor &src, const float scale, const float offset, - cudaStream_t stream) -{ - NormalizeC1Batch(dst, const_cast &>(src), scale, offset, stream); -} - -void Normalize(Tensor &dst, const Tensor &src, const float scale, const float offset, - cudaStream_t stream) -{ - NormalizeC1F32Impl(dst, src, scale, offset, stream); -} - -void Normalize(Tensor &dst, const Tensor &src, const float scale, const float offset, - cudaStream_t stream) -{ - NormalizeC1Batch(dst, const_cast &>(src), scale, offset, stream); -} - -}} // namespace cvcore::tensor_ops \ No newline at end of file diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/BBoxUtils.cpp b/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/BBoxUtils.cpp deleted file mode 100644 index 4559a35..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/BBoxUtils.cpp +++ /dev/null @@ -1,173 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2020-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -#include "cv/tensor_ops/BBoxUtils.h" - -#include -#include - -namespace cvcore { namespace tensor_ops { - -namespace { - -bool IsValid(const BBox &box) -{ - return box.xmin >= 0 && box.ymin >= 0 && box.xmin < box.xmax && box.ymin < box.ymax; -} - -} // anonymous namespace - -float GetArea(const BBox &box) -{ - if (box.xmax < box.xmin || box.ymax < box.ymin) - { - return 0.f; - } - return static_cast((box.xmax - box.xmin) * (box.ymax - box.ymin)); -} - -float GetIntersection(const BBox &a, const BBox &b) -{ - const int lowerX = std::max(a.xmin, b.xmin); - const int upperX = std::min(a.xmax, b.xmax); - const int lowerY = std::max(a.ymin, b.ymin); - const int upperY = std::min(a.ymax, b.ymax); - const int diffX = lowerX < upperX ? upperX - lowerX : 0; - const int diffY = lowerY < upperY ? upperY - lowerY : 0; - return static_cast(diffX * diffY); -} - -float GetUnion(const BBox &a, const BBox &b) -{ - return GetArea(a) + GetArea(b) - GetIntersection(a, b); -} - -float GetIoU(const BBox &a, const BBox &b) -{ - return GetIntersection(a, b) / GetUnion(a, b); -} - -BBox MergeBoxes(const BBox &a, const BBox &b) -{ - if (!IsValid(a) || !IsValid(b)) - { - return IsValid(a) ? a : b; - } - BBox res; - res.xmin = std::min(a.xmin, b.xmin); - res.xmax = std::max(a.xmax, b.xmax); - res.ymin = std::min(a.ymin, b.ymin); - res.ymax = std::max(a.ymax, b.ymax); - return res; -} - -BBox ClampBox(const BBox &a, const BBox &b) -{ - return {std::max(a.xmin, b.xmin), std::max(a.ymin, b.ymin), std::min(a.xmax, b.xmax), std::min(a.ymax, b.ymax)}; -} - -BBox InterpolateBoxes(float currLeft, float currRight, float currBottom, float currTop, float xScaler, float yScaler, - int currColumn, int currRow, BBoxInterpolationType type, float bboxNorm) -{ - BBox currBoxInfo; - if (type == CONST_INTERPOLATION) - { - float centerX = ((currColumn * xScaler + 0.5) / bboxNorm); - float centerY = ((currRow * yScaler + 0.5) / bboxNorm); - float left = (currLeft - centerX); - float right = (currRight + centerX); - float top = (currTop - centerY); - float bottom = (currBottom + centerY); - currBoxInfo.xmin = left * -bboxNorm; - currBoxInfo.xmax = right * bboxNorm; - currBoxInfo.ymin = top * -bboxNorm; - currBoxInfo.ymax = bottom * bboxNorm; - } - else if (type == IMAGE_INTERPOLATION) - { - int centerX = (int)((currColumn + 0.5f) * xScaler); - int centerY = (int)((currRow + 0.5f) * yScaler); - int left = (int)(currLeft * xScaler); - int right = (int)(currRight * xScaler); - int top = (int)(currTop * yScaler); - int bottom = (int)(currBottom * yScaler); - currBoxInfo.xmin = centerX - left; - currBoxInfo.xmax = centerX + right; - currBoxInfo.ymin = centerY - top; - currBoxInfo.ymax = centerY + bottom; - } - else - { - throw std::runtime_error("invalid bbox interpolation type"); - } - return currBoxInfo; -} - -BBox ScaleBox(const BBox &bbox, float xScaler, float yScaler, BBoxScaleType type) -{ - BBox output; - if (type == NORMAL) - { - int xMin = (int)(bbox.xmin * xScaler + 0.5f); - int yMin = (int)(bbox.ymin * yScaler + 0.5f); - int xMax = (int)(bbox.xmax * xScaler + 0.5f); - int yMax = (int)(bbox.ymax * yScaler + 0.5f); - output = {xMin, yMin, xMax, yMax}; - } - else if (type == CENTER) - { - float xCenter = (bbox.xmax + bbox.xmin) / 2.0f; - float yCenter = (bbox.ymax + bbox.ymin) / 2.0f; - - float width = (bbox.xmax - bbox.xmin) * xScaler; - float height = (bbox.ymax - bbox.ymin) * yScaler; - - output = {int(xCenter - width / 2 + 0.5f), int(yCenter - height / 2 + 0.5f), int(xCenter + width / 2 + 0.5f), - int(yCenter + height / 2 + 0.5f)}; - } - else - { - throw std::runtime_error("invalid bbox scaling type"); - } - return output; -} - -BBox TransformBox(const BBox &bbox, float xScaler, float yScaler, float xOffset, float yOffset) -{ - int xMin = (int)((bbox.xmin + xOffset) * xScaler + 0.5f); - int yMin = (int)((bbox.ymin + yOffset) * yScaler + 0.5f); - int xMax = (int)((bbox.xmax + xOffset) * xScaler + 0.5f); - int yMax = (int)((bbox.ymax + yOffset) * yScaler + 0.5f); - return {xMin, yMin, xMax, yMax}; -} - -BBox SquarifyBox(const BBox &box, const BBox &boundary, float scale) -{ - BBox output = ClampBox(box, boundary); - float updateWH = scale * std::max(output.xmax - output.xmin, output.ymax - output.ymin); - float scaleW = updateWH / float(output.xmax - output.xmin); - float scaleH = updateWH / float(output.ymax - output.ymin); - output = ScaleBox(output, scaleW, scaleH, CENTER); - - output = ClampBox(output, boundary); - int xmin = output.xmin; - int ymin = output.ymin; - int l = std::min(output.xmax - output.xmin, output.ymax - output.ymin); - return {xmin, ymin, xmin + l, ymin + l}; -} - -}} // namespace cvcore::tensor_ops diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/ColorConversions.cpp b/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/ColorConversions.cpp deleted file mode 100644 index f02d3a9..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/ColorConversions.cpp +++ /dev/null @@ -1,447 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2020-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -#include "NppUtils.h" - -#include "cv/tensor_ops/ImageUtils.h" - -#include "cv/core/Memory.h" - -#include -#include -#include - -#include -#include -#include - -namespace cvcore { namespace tensor_ops { - -const float BGR2GRAY_COEFFS[3] = {0.114f, 0.587f, 0.299f}; -const float RGB2GRAY_COEFFS[3] = {0.299f, 0.587f, 0.114f}; - -namespace { - -template -void ConvertColorFormatBatch(Tensor &dst, Tensor &src, ColorConversionType type, - cudaStream_t stream) -{ - // src and dst must be GPU tensors - assert(!src.isCPU() && !dst.isCPU()); - assert(src.getDepth() == dst.getDepth()); - - for (int i = 0; i < src.getDepth(); i++) - { - size_t offsetSrc = i * src.getStride(TensorDimension::DEPTH); - size_t offsetDst = i * dst.getStride(TensorDimension::DEPTH); - Tensor srcTmp(src.getWidth(), src.getHeight(), - src.getStride(TensorDimension::HEIGHT) * GetChannelSize(CT), - src.getData() + offsetSrc, false); - Tensor dstTmp(dst.getWidth(), dst.getHeight(), - dst.getStride(TensorDimension::HEIGHT) * GetChannelSize(CT), - dst.getData() + offsetDst, false); - ConvertColorFormat(dstTmp, srcTmp, type, stream); - } -} - -template -void InterleavedToPlanarBatch(Tensor &dst, Tensor &src, cudaStream_t stream) -{ - // src and dst must be GPU tensors - assert(!src.isCPU() && !dst.isCPU()); - assert(src.getDepth() == dst.getDepth()); - - for (int i = 0; i < src.getDepth(); i++) - { - size_t offsetSrc = i * src.getStride(TensorDimension::DEPTH); - size_t offsetDst = i * dst.getStride(TensorDimension::DEPTH); - Tensor srcTmp(src.getWidth(), src.getHeight(), - src.getStride(TensorDimension::HEIGHT) * GetChannelSize(CT), - src.getData() + offsetSrc, false); - Tensor dstTmp(dst.getWidth(), dst.getHeight(), - dst.getStride(TensorDimension::HEIGHT) * GetChannelSize(CT), - dst.getData() + offsetDst, false); - InterleavedToPlanar(dstTmp, srcTmp, stream); - } -} - -} // anonymous namespace - -void ConvertColorFormat(Tensor &dst, const Tensor &src, ColorConversionType type, - cudaStream_t stream) -{ - // src and dst must be GPU tensors - assert(!src.isCPU() && !dst.isCPU()); - assert((src.getWidth() == dst.getWidth()) && (src.getHeight() == dst.getHeight())); - - if (type == BGR2RGB || type == RGB2BGR) - { - const int order[3] = {2, 1, 0}; - NppStatus status = nppiSwapChannels_8u_C3R_Ctx( - static_cast(src.getData()), src.getStride(TensorDimension::HEIGHT) * sizeof(Npp8u), - static_cast(dst.getData()), dst.getStride(TensorDimension::HEIGHT) * sizeof(Npp8u), - {int(src.getWidth()), int(src.getHeight())}, order, GetNppStreamContext(stream)); - assert(status == NPP_SUCCESS); - } - else - { - throw std::runtime_error("invalid color conversion type"); - } -} - -void ConvertColorFormat(Tensor &dst, const Tensor &src, ColorConversionType type, - cudaStream_t stream) -{ - ConvertColorFormatBatch(dst, const_cast &>(src), type, stream); -} - -void ConvertColorFormat(Tensor &dst, const Tensor &src, ColorConversionType type, - cudaStream_t stream) -{ - // src and dst must be GPU tensors - assert(!src.isCPU() && !dst.isCPU()); - assert((src.getWidth() == dst.getWidth()) && (src.getHeight() == dst.getHeight())); - - if (type == BGR2RGB || type == RGB2BGR) - { - const int order[3] = {2, 1, 0}; - NppStatus status = nppiSwapChannels_16u_C3R_Ctx( - static_cast(src.getData()), src.getStride(TensorDimension::HEIGHT) * sizeof(Npp16u), - static_cast(dst.getData()), dst.getStride(TensorDimension::HEIGHT) * sizeof(Npp16u), - {int(src.getWidth()), int(src.getHeight())}, order, GetNppStreamContext(stream)); - assert(status == NPP_SUCCESS); - } - else - { - throw std::runtime_error("invalid color conversion type"); - } -} - -void ConvertColorFormat(Tensor &dst, const Tensor &src, ColorConversionType type, - cudaStream_t stream) -{ - ConvertColorFormatBatch(dst, const_cast &>(src), type, stream); -} - -void ConvertColorFormat(Tensor &dst, const Tensor &src, ColorConversionType type, - cudaStream_t stream) -{ - // src and dst must be GPU tensors - assert(!src.isCPU() && !dst.isCPU()); - assert((src.getWidth() == dst.getWidth()) && (src.getHeight() == dst.getHeight())); - - if (type == BGR2RGB || type == RGB2BGR) - { - const int order[3] = {2, 1, 0}; - NppStatus status = nppiSwapChannels_32f_C3R_Ctx( - static_cast(src.getData()), src.getStride(TensorDimension::HEIGHT) * sizeof(Npp32f), - static_cast(dst.getData()), dst.getStride(TensorDimension::HEIGHT) * sizeof(Npp32f), - {int(src.getWidth()), int(src.getHeight())}, order, GetNppStreamContext(stream)); - assert(status == NPP_SUCCESS); - } - else - { - throw std::runtime_error("invalid color conversion type"); - } -} - -void ConvertColorFormat(Tensor &dst, const Tensor &src, ColorConversionType type, - cudaStream_t stream) -{ - ConvertColorFormatBatch(dst, const_cast &>(src), type, stream); -} - -void ConvertColorFormat(Tensor &dst, const Tensor &src, ColorConversionType type, - cudaStream_t stream) -{ - // src and dst must be GPU tensors - assert(!src.isCPU() && !dst.isCPU()); - assert((src.getWidth() == dst.getWidth()) && (src.getHeight() == dst.getHeight())); - - if (type == BGR2GRAY || type == RGB2GRAY) - { - NppStatus status = nppiColorToGray_8u_C3C1R_Ctx( - static_cast(src.getData()), src.getStride(TensorDimension::HEIGHT) * sizeof(Npp8u), - static_cast(dst.getData()), dst.getStride(TensorDimension::HEIGHT) * sizeof(Npp8u), - {int(src.getWidth()), int(src.getHeight())}, type == BGR2GRAY ? BGR2GRAY_COEFFS : RGB2GRAY_COEFFS, - GetNppStreamContext(stream)); - assert(status == NPP_SUCCESS); - } - else - { - throw std::runtime_error("invalid color conversion type"); - } -} - -void ConvertColorFormat(Tensor &dst, const Tensor &src, ColorConversionType type, - cudaStream_t stream) -{ - // src and dst must be GPU tensors - assert(!src.isCPU() && !dst.isCPU()); - assert((src.getWidth() == dst.getWidth()) && (src.getHeight() == dst.getHeight())); - - if (type == BGR2GRAY || type == RGB2GRAY) - { - NppStatus status = nppiColorToGray_16u_C3C1R_Ctx( - static_cast(src.getData()), src.getStride(TensorDimension::HEIGHT) * sizeof(Npp16u), - static_cast(dst.getData()), dst.getStride(TensorDimension::HEIGHT) * sizeof(Npp16u), - {int(src.getWidth()), int(src.getHeight())}, type == BGR2GRAY ? BGR2GRAY_COEFFS : RGB2GRAY_COEFFS, - GetNppStreamContext(stream)); - assert(status == NPP_SUCCESS); - } - else - { - throw std::runtime_error("invalid color conversion type"); - } -} - -void ConvertColorFormat(Tensor &dst, const Tensor &src, ColorConversionType type, - cudaStream_t stream) -{ - // src and dst must be GPU tensors - assert(!src.isCPU() && !dst.isCPU()); - assert((src.getWidth() == dst.getWidth()) && (src.getHeight() == dst.getHeight())); - - if (type == BGR2GRAY || type == RGB2GRAY) - { - NppStatus status = nppiColorToGray_32f_C3C1R_Ctx( - static_cast(src.getData()), src.getStride(TensorDimension::HEIGHT) * sizeof(Npp32f), - static_cast(dst.getData()), dst.getStride(TensorDimension::HEIGHT) * sizeof(Npp32f), - {int(src.getWidth()), int(src.getHeight())}, type == BGR2GRAY ? BGR2GRAY_COEFFS : RGB2GRAY_COEFFS, - GetNppStreamContext(stream)); - assert(status == NPP_SUCCESS); - } - else - { - throw std::runtime_error("invalid color conversion type"); - } -} - -void ConvertColorFormat(Tensor &dst, const Tensor &src, ColorConversionType type, - cudaStream_t stream) -{ - // src and dst must be GPU tensors - assert(!src.isCPU() && !dst.isCPU()); - assert((src.getWidth() == dst.getWidth()) && (src.getHeight() == dst.getHeight())); - - if (type == GRAY2BGR || type == GRAY2RGB) - { - NppStatus status = nppiDup_8u_C1C3R_Ctx( - static_cast(src.getData()), src.getStride(TensorDimension::HEIGHT) * sizeof(Npp8u), - static_cast(dst.getData()), dst.getStride(TensorDimension::HEIGHT) * sizeof(Npp8u), - {int(src.getWidth()), int(src.getHeight())}, GetNppStreamContext(stream)); - assert(status == NPP_SUCCESS); - } - else - { - throw std::runtime_error("invalid color conversion type"); - } -} - -void ConvertColorFormat(Tensor &dst, const Tensor &src, ColorConversionType type, - cudaStream_t stream) -{ - // src and dst must be GPU tensors - assert(!src.isCPU() && !dst.isCPU()); - assert((src.getWidth() == dst.getWidth()) && (src.getHeight() == dst.getHeight())); - - if (type == GRAY2BGR || type == GRAY2RGB) - { - NppStatus status = nppiDup_16u_C1C3R_Ctx( - static_cast(src.getData()), src.getStride(TensorDimension::HEIGHT) * sizeof(Npp16u), - static_cast(dst.getData()), dst.getStride(TensorDimension::HEIGHT) * sizeof(Npp16u), - {int(src.getWidth()), int(src.getHeight())}, GetNppStreamContext(stream)); - assert(status == NPP_SUCCESS); - } - else - { - throw std::runtime_error("invalid color conversion type"); - } -} - -void ConvertColorFormat(Tensor &dst, const Tensor &src, ColorConversionType type, - cudaStream_t stream) -{ - // src and dst must be GPU tensors - assert(!src.isCPU() && !dst.isCPU()); - assert((src.getWidth() == dst.getWidth()) && (src.getHeight() == dst.getHeight())); - - if (type == GRAY2BGR || type == GRAY2RGB) - { - NppStatus status = nppiDup_32f_C1C3R_Ctx( - static_cast(src.getData()), src.getStride(TensorDimension::HEIGHT) * sizeof(Npp32f), - static_cast(dst.getData()), dst.getStride(TensorDimension::HEIGHT) * sizeof(Npp32f), - {int(src.getWidth()), int(src.getHeight())}, GetNppStreamContext(stream)); - assert(status == NPP_SUCCESS); - } - else - { - throw std::runtime_error("invalid color conversion type"); - } -} - -void ConvertBitDepth(Tensor &dst, Tensor &src, const float scale, cudaStream_t stream) -{ - // src and dst must be GPU tensors - assert(!src.isCPU() && !dst.isCPU()); - assert((src.getWidth() == dst.getWidth()) && (src.getHeight() == dst.getHeight())); - - const NppiSize srcSize = {src.getWidth(), src.getHeight()}; - - NppStreamContext streamContext = GetNppStreamContext(stream); - - NppStatus status = - nppiMulC_32f_C1IR_Ctx(static_cast(scale), static_cast(src.getData()), - src.getStride(TensorDimension::HEIGHT) * sizeof(Npp32f), srcSize, streamContext); - assert(status == NPP_SUCCESS); - - status = nppiConvert_32f8u_C1R_Ctx( - static_cast(src.getData()), src.getStride(TensorDimension::HEIGHT) * sizeof(Npp32f), - static_cast(dst.getData()), dst.getStride(TensorDimension::HEIGHT) * sizeof(Npp8u), - {int(src.getWidth()), int(src.getHeight())}, NPP_RND_FINANCIAL, streamContext); - assert(status == NPP_SUCCESS); -} - -void ConvertBitDepth(Tensor &dst, Tensor &src, const float scale, cudaStream_t stream) -{ - // src and dst must be GPU tensors - assert(!src.isCPU() && !dst.isCPU()); - assert(src.getDepth() == dst.getDepth()); - - Tensor srcTmp(src.getWidth(), src.getDepth() * src.getHeight(), - src.getStride(TensorDimension::HEIGHT) * GetChannelSize(F32), src.getData(), false); - Tensor dstTmp(dst.getWidth(), dst.getDepth() * dst.getHeight(), - dst.getStride(TensorDimension::HEIGHT) * GetChannelSize(U8), dst.getData(), false); - ConvertBitDepth(dstTmp, srcTmp, scale, stream); -} - -void InterleavedToPlanar(Tensor &dst, const Tensor &src, cudaStream_t stream) -{ - // src and dst must be GPU tensors - assert(!src.isCPU() && !dst.isCPU()); - assert((src.getWidth() == dst.getWidth()) && (src.getHeight() == dst.getHeight())); - - NppStatus status; - NppStreamContext streamContext = GetNppStreamContext(stream); - - const size_t offset = dst.getStride(TensorDimension::HEIGHT) * dst.getHeight(); - Npp8u *const dstBuffer[3] = {dst.getData(), dst.getData() + offset, dst.getData() + 2 * offset}; - status = nppiCopy_8u_C3P3R_Ctx(static_cast(src.getData()), - src.getStride(TensorDimension::HEIGHT) * sizeof(Npp8u), dstBuffer, - dst.getStride(TensorDimension::HEIGHT) * sizeof(Npp8u), - {int(src.getWidth()), int(src.getHeight())}, streamContext); - assert(status == NPP_SUCCESS); -} - -void InterleavedToPlanar(Tensor &dst, const Tensor &src, cudaStream_t stream) -{ - InterleavedToPlanarBatch(dst, const_cast &>(src), stream); -} - -void InterleavedToPlanar(Tensor &dst, const Tensor &src, cudaStream_t stream) -{ - // src and dst must be GPU tensors - assert(!src.isCPU() && !dst.isCPU()); - assert((src.getWidth() == dst.getWidth()) && (src.getHeight() == dst.getHeight())); - - NppStatus status; - NppStreamContext streamContext = GetNppStreamContext(stream); - - const size_t offset = dst.getStride(TensorDimension::HEIGHT) * dst.getHeight(); - Npp16u *const dstBuffer[3] = {dst.getData(), dst.getData() + offset, dst.getData() + 2 * offset}; - status = nppiCopy_16u_C3P3R_Ctx(static_cast(src.getData()), - src.getStride(TensorDimension::HEIGHT) * sizeof(Npp16u), dstBuffer, - dst.getStride(TensorDimension::HEIGHT) * sizeof(Npp16u), - {int(src.getWidth()), int(src.getHeight())}, streamContext); -} - -void InterleavedToPlanar(Tensor &dst, const Tensor &src, cudaStream_t stream) -{ - InterleavedToPlanarBatch(dst, const_cast &>(src), stream); -} - -void InterleavedToPlanar(Tensor &dst, const Tensor &src, cudaStream_t stream) -{ - // src and dst must be GPU tensors - assert(!src.isCPU() && !dst.isCPU()); - assert((src.getWidth() == dst.getWidth()) && (src.getHeight() == dst.getHeight())); - - NppStatus status; - NppStreamContext streamContext = GetNppStreamContext(stream); - - const size_t offset = dst.getStride(TensorDimension::HEIGHT) * dst.getHeight(); - Npp32f *const dstBuffer[3] = {dst.getData(), dst.getData() + offset, dst.getData() + 2 * offset}; - status = nppiCopy_32f_C3P3R_Ctx(static_cast(src.getData()), - src.getStride(TensorDimension::HEIGHT) * sizeof(Npp32f), dstBuffer, - dst.getStride(TensorDimension::HEIGHT) * sizeof(Npp32f), - {int(src.getWidth()), int(src.getHeight())}, streamContext); -} - -void InterleavedToPlanar(Tensor &dst, const Tensor &src, cudaStream_t stream) -{ - InterleavedToPlanarBatch(dst, const_cast &>(src), stream); -} - -void InterleavedToPlanar(Tensor &dst, const Tensor &src, cudaStream_t stream) -{ - // src and dst must be GPU tensors - assert(!src.isCPU() && !dst.isCPU()); - assert((src.getWidth() == dst.getWidth()) && (src.getHeight() == dst.getHeight())); - - Tensor tmp(dst.getWidth(), dst.getHeight(), dst.getStride(TensorDimension::HEIGHT) * sizeof(Npp8u), - dst.getData(), false); - Copy(tmp, src, stream); -} - -void InterleavedToPlanar(Tensor &dst, const Tensor &src, cudaStream_t stream) -{ - InterleavedToPlanarBatch(dst, const_cast &>(src), stream); -} - -void InterleavedToPlanar(Tensor &dst, const Tensor &src, cudaStream_t stream) -{ - // src and dst must be GPU tensors - assert(!src.isCPU() && !dst.isCPU()); - assert((src.getWidth() == dst.getWidth()) && (src.getHeight() == dst.getHeight())); - - Tensor tmp(dst.getWidth(), dst.getHeight(), dst.getStride(TensorDimension::HEIGHT) * sizeof(Npp16u), - dst.getData(), false); - Copy(tmp, src, stream); -} - -void InterleavedToPlanar(Tensor &dst, const Tensor &src, cudaStream_t stream) -{ - InterleavedToPlanarBatch(dst, const_cast &>(src), stream); -} - -void InterleavedToPlanar(Tensor &dst, const Tensor &src, cudaStream_t stream) -{ - // src and dst must be GPU tensors - assert(!src.isCPU() && !dst.isCPU()); - assert((src.getWidth() == dst.getWidth()) && (src.getHeight() == dst.getHeight())); - - Tensor tmp(dst.getWidth(), dst.getHeight(), dst.getStride(TensorDimension::HEIGHT) * sizeof(Npp32f), - dst.getData(), false); - Copy(tmp, src, stream); -} - -void InterleavedToPlanar(Tensor &dst, const Tensor &src, cudaStream_t stream) -{ - InterleavedToPlanarBatch(dst, const_cast &>(src), stream); -} - -}} // namespace cvcore::tensor_ops diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/DBScan.cpp b/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/DBScan.cpp deleted file mode 100644 index 0a3c2ea..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/DBScan.cpp +++ /dev/null @@ -1,214 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2020-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -#include "cv/tensor_ops/DBScan.h" -#include "cv/tensor_ops/BBoxUtils.h" - -#include -#include - -namespace cvcore { namespace tensor_ops { - -constexpr int kUnclassified = -1; -constexpr int kCorePoint = 1; -constexpr int kBorderPoint = 2; -constexpr int kNoise = -2; - -namespace { - -float CalculateDistance(const BBox &lhs, const BBox &rhs) -{ - const float iou = GetIoU(lhs, rhs); - return 1.0f - iou; -} - -void MergeMaximumBBoxes(Array &input, Array &clusters, Array &output) -{ - BBox tempBox = {-1, -1, -1, -1}; - // Initialize each cluster-box with a placeholder that has no cluster - for (int i = 0; i < output.getSize(); i++) - { - // It's a struct so these pushes are by value - output[i] = tempBox; - } - - for (int i = 0; i < input.getSize(); i++) - { - int clusterId = clusters[i]; - if (clusterId >= 0) - { - // Box merging is associative & commutative - output[clusterId] = MergeBoxes(input[i], output[clusterId]); - } - } -} - -void MergeWeightedBBoxes(Array &input, Array &clusters, Array &weights, Array &output) -{ - int numClusters = output.getSize(); - // centos has gcc 4.8.5 which complains about initializing variable sized arrays with {}. - // Use std::vector for variable sized array. - std::vector xmins(numClusters, 0); - std::vector ymins(numClusters, 0); - std::vector xmaxs(numClusters, 0); - std::vector ymaxs(numClusters, 0); - std::vector scales(numClusters, 0); - - for (int i = 0; i < input.getSize(); i++) - { - int clusterId = clusters[i]; - if (clusterId >= 0) - { - xmins[clusterId] += input[i].xmin * weights[i]; - ymins[clusterId] += input[i].ymin * weights[i]; - xmaxs[clusterId] += input[i].xmax * weights[i]; - ymaxs[clusterId] += input[i].ymax * weights[i]; - scales[clusterId] += weights[i]; - } - } - - for (int i = 0; i < numClusters; i++) - { - output[i] = {int(xmins[i] / scales[i] + 0.5f), int(ymins[i] / scales[i] + 0.5f), - int(xmaxs[i] / scales[i] + 0.5f), int(ymaxs[i] / scales[i] + 0.5f)}; - } -} - -} // anonymous namespace - -DBScan::DBScan(int pointsSize, int minPoints, float epsilon) - : m_pointsSize(pointsSize) - , m_numClusters(0) - , m_minPoints(minPoints) - , m_epsilon(epsilon) - , m_clusterStates(pointsSize, true) -{ - m_clusterStates.setSize(pointsSize); -} - -void DBScan::doCluster(Array &input, Array &clusters) -{ - // Reset all cluster id - for (int i = 0; i < m_pointsSize; i++) - { - clusters[i] = -1; - m_clusterStates[i] = kUnclassified; - } - int nextClusterId = 0; - for (int cIndex = 0; cIndex < m_pointsSize; cIndex++) - { - std::vector neighbors; - for (int neighborIndex = 0; neighborIndex < m_pointsSize; neighborIndex++) - { - if (neighborIndex == cIndex) - { - continue; // Don't look at being your own neighbor - } - if (CalculateDistance(input[cIndex], input[neighborIndex]) <= m_epsilon) - { - // nrighborIndex is in our neighborhood - neighbors.push_back(neighborIndex); - - if (m_clusterStates[neighborIndex] == kCorePoint) - { - // We are at the neighborhood of a core point, we are at least a border point - m_clusterStates[cIndex] = kBorderPoint; - // Take the first cluster number as you can - if (clusters[cIndex] == -1) - { - clusters[cIndex] = clusters[neighborIndex]; - } - } - } - } - if (neighbors.size() >= m_minPoints - 1) - { - m_clusterStates[cIndex] = kCorePoint; - if (clusters[cIndex] == -1) - { - // We're not in the neighborhood of other core points - // So we're the core of a new cluster - clusters[cIndex] = nextClusterId; - nextClusterId++; - } - - // Set all neighbors that came before us to be border points - for (int neighborListIndex = 0; - neighborListIndex < neighbors.size() && neighbors[neighborListIndex] < cIndex; neighborListIndex++) - { - if (m_clusterStates[neighbors[neighborListIndex]] == kNoise) - { - // If it was noise, now it's a border point in our cluster - m_clusterStates[neighbors[neighborListIndex]] = kBorderPoint; - // Make sure everything that's in our neighborhood is our cluster id - clusters[neighbors[neighborListIndex]] = clusters[cIndex]; - } - } - } - else - { - // We are a border point, or a noise point - if (m_clusterStates[cIndex] == kUnclassified) - { - m_clusterStates[cIndex] = kNoise; - clusters[cIndex] = -1; - } - } - } - - m_numClusters = nextClusterId; // Number of clusters -} - -void DBScan::doClusterAndMerge(Array &input, Array &output, BBoxMergeType type) -{ - Array clusters(m_pointsSize, true); - clusters.setSize(m_pointsSize); - doCluster(input, clusters); - output.setSize(m_numClusters); - - // merge bboxes based on different modes (TODO: might add mininum/average in the future) - if (type == MAXIMUM) - { - MergeMaximumBBoxes(input, clusters, output); - } - else - { - throw std::runtime_error("Unsupported bbox merge type."); - } -} - -void DBScan::doClusterAndMerge(Array &input, Array &weights, Array &output, BBoxMergeType type) -{ - Array clusters(m_pointsSize, true); - clusters.setSize(m_pointsSize); - doCluster(input, clusters); - output.setSize(m_numClusters); - - // merge type must be WEIGHTED - if (type != WEIGHTED) - { - throw std::runtime_error("Bbox merge type must be WEIGHTED."); - } - MergeWeightedBBoxes(input, clusters, weights, output); -} - -int DBScan::getNumClusters() const -{ - return m_numClusters; -} - -}} // namespace cvcore::tensor_ops diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/Errors.cpp b/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/Errors.cpp deleted file mode 100644 index d29a1ae..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/Errors.cpp +++ /dev/null @@ -1,104 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -#include "cv/tensor_ops/Errors.h" - -#ifndef __cpp_lib_to_underlying -// Using a C++23 feature by hacking std -namespace std -{ - template - constexpr underlying_type_t to_underlying(Enum e) noexcept - { - return static_cast>(e); - } -}; -#endif // __cpp_lib_to_underlying - -namespace cvcore { namespace tensor_ops { - -namespace detail -{ - struct TensorOpsErrorCategory : std::error_category - { - virtual const char * name() const noexcept override final - { - return "cvcore-tensor-ops-error"; - } - - virtual std::string message(int value) const override final - { - std::string result; - - switch(value) - { - case std::to_underlying(TensorOpsErrorCode::SUCCESS): - result = "(SUCCESS) No errors detected"; - break; - case std::to_underlying(TensorOpsErrorCode::COMPUTE_ENGINE_UNSUPPORTED_BY_CONTEXT): - result = "(COMPUTE_ENGINE_UNSUPPORTED_BY_CONTEXT) The selected compute " - "engine defined by cvcore::ComputeEngine is not avaible in the " - "requested context defined by cvcore::tensor_ops::TensorBackend"; - break; - case std::to_underlying(TensorOpsErrorCode::CAMERA_DISTORTION_MODEL_UNSUPPORTED): - result = "(CAMERA_DISTORTION_MODEL_UNSUPPORTED) The selected camera " - "distortion model defined by cvcore::CameraDistortionType is " - "currently unsupported"; - break; - default: - result = "(Unrecognized Condition) Value " + std::to_string(value) + - " does not map to known error code literal " + - " defined by cvcore::tensor_ops::TensorOpsErrorCode"; - break; - } - - return result; - } - - virtual std::error_condition default_error_condition(int code) const noexcept override final - { - std::error_condition result; - - switch(code) - { - case std::to_underlying(TensorOpsErrorCode::SUCCESS): - result = ErrorCode::SUCCESS; - break; - case std::to_underlying(TensorOpsErrorCode::COMPUTE_ENGINE_UNSUPPORTED_BY_CONTEXT): - result = ErrorCode::INVALID_ENGINE_TYPE; - break; - case std::to_underlying(TensorOpsErrorCode::CAMERA_DISTORTION_MODEL_UNSUPPORTED): - result = ErrorCode::INVALID_ARGUMENT; - break; - default: - result = ErrorCode::NOT_IMPLEMENTED; - break; - } - - return result; - } - }; -} // namespace detail - -const detail::TensorOpsErrorCategory errorCategory{}; - -std::error_code make_error_code(TensorOpsErrorCode ec) noexcept -{ - return {std::to_underlying(ec), errorCategory}; -} - -}} // namespace cvcore::tensor_ops diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/Filters.cpp b/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/Filters.cpp deleted file mode 100644 index d8bd75c..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/Filters.cpp +++ /dev/null @@ -1,112 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2020-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -#include "Filters.h" -#include "NppUtils.h" - -#include "cv/core/MathTypes.h" -#include "cv/core/Memory.h" - -#include - -#include -#include -#include - -namespace cvcore { namespace tensor_ops { - -void BoxFilter(Tensor &dst, const Tensor &src, const Vector2i &maskSize, - const Vector2i &anchor, cudaStream_t stream) -{ - assert(!src.isCPU() && !dst.isCPU()); - NppStatus status = nppiFilterBoxBorder_8u_C3R_Ctx( - static_cast(src.getData()), src.getStride(TensorDimension::HEIGHT) * sizeof(Npp8u), - {static_cast(src.getWidth()), static_cast(src.getHeight())}, {0, 0}, - static_cast(dst.getData()), dst.getStride(TensorDimension::HEIGHT) * sizeof(Npp8u), - {static_cast(src.getWidth()), static_cast(src.getHeight())}, {maskSize.x, maskSize.y}, - {anchor.x, anchor.y}, - NPP_BORDER_REPLICATE, //Only Npp Replicate is supported!!! - GetNppStreamContext(stream)); - assert(status == NPP_SUCCESS); -} - -void BoxFilter(Tensor &dst, const Tensor &src, const Vector2i &maskSize, - const Vector2i &anchor, cudaStream_t stream) -{ - assert(!src.isCPU() && !dst.isCPU()); - NppStatus status = nppiFilterBoxBorder_8u_C1R_Ctx( - static_cast(src.getData()), src.getStride(TensorDimension::HEIGHT) * sizeof(Npp8u), - {static_cast(src.getWidth()), static_cast(src.getHeight())}, {0, 0}, - static_cast(dst.getData()), dst.getStride(TensorDimension::HEIGHT) * sizeof(Npp8u), - {static_cast(src.getWidth()), static_cast(src.getHeight())}, {maskSize.x, maskSize.y}, - {anchor.x, anchor.y}, NPP_BORDER_REPLICATE, GetNppStreamContext(stream)); - assert(status == NPP_SUCCESS); -} - -void BoxFilter(Tensor &dst, const Tensor &src, const Vector2i &maskSize, - const Vector2i &anchor, cudaStream_t stream) -{ - assert(!src.isCPU() && !dst.isCPU()); - NppStatus status = nppiFilterBoxBorder_16u_C1R_Ctx( - static_cast(src.getData()), src.getStride(TensorDimension::HEIGHT) * sizeof(Npp16u), - {static_cast(src.getWidth()), static_cast(src.getHeight())}, {0, 0}, - static_cast(dst.getData()), dst.getStride(TensorDimension::HEIGHT) * sizeof(Npp16u), - {static_cast(src.getWidth()), static_cast(src.getHeight())}, {maskSize.x, maskSize.y}, - {anchor.x, anchor.y}, NPP_BORDER_REPLICATE, GetNppStreamContext(stream)); - assert(status == NPP_SUCCESS); -} - -void BoxFilter(Tensor &dst, const Tensor &src, const Vector2i &maskSize, - const Vector2i &anchor, cudaStream_t stream) -{ - assert(!src.isCPU() && !dst.isCPU()); - NppStatus status = nppiFilterBoxBorder_16u_C3R_Ctx( - static_cast(src.getData()), src.getStride(TensorDimension::HEIGHT) * sizeof(Npp16u), - {static_cast(src.getWidth()), static_cast(src.getHeight())}, {0, 0}, - static_cast(dst.getData()), dst.getStride(TensorDimension::HEIGHT) * sizeof(Npp16u), - {static_cast(src.getWidth()), static_cast(src.getHeight())}, {maskSize.x, maskSize.y}, - {anchor.x, anchor.y}, NPP_BORDER_REPLICATE, GetNppStreamContext(stream)); - assert(status == NPP_SUCCESS); -} - -void BoxFilter(Tensor &dst, const Tensor &src, const Vector2i &maskSize, - const Vector2i &anchor, cudaStream_t stream) -{ - assert(!src.isCPU() && !dst.isCPU()); - NppStatus status = nppiFilterBoxBorder_32f_C3R_Ctx( - static_cast(src.getData()), src.getStride(TensorDimension::HEIGHT) * sizeof(Npp32f), - {static_cast(src.getWidth()), static_cast(src.getHeight())}, {0, 0}, - static_cast(dst.getData()), dst.getStride(TensorDimension::HEIGHT) * sizeof(Npp32f), - {static_cast(src.getWidth()), static_cast(src.getHeight())}, {maskSize.x, maskSize.y}, - {anchor.x, anchor.y}, NPP_BORDER_REPLICATE, GetNppStreamContext(stream)); - assert(status == NPP_SUCCESS); -} - -void BoxFilter(Tensor &dst, const Tensor &src, const Vector2i &maskSize, - const Vector2i &anchor, cudaStream_t stream) -{ - assert(!src.isCPU() && !dst.isCPU()); - NppStatus status = nppiFilterBoxBorder_32f_C1R_Ctx( - static_cast(src.getData()), src.getStride(TensorDimension::HEIGHT) * sizeof(Npp32f), - {static_cast(src.getWidth()), static_cast(src.getHeight())}, {0, 0}, - static_cast(dst.getData()), dst.getStride(TensorDimension::HEIGHT) * sizeof(Npp32f), - {static_cast(src.getWidth()), static_cast(src.getHeight())}, {maskSize.x, maskSize.y}, - {anchor.x, anchor.y}, NPP_BORDER_REPLICATE, GetNppStreamContext(stream)); - assert(status == NPP_SUCCESS); -} - -}} // namespace cvcore::tensor_ops diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/Filters.h b/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/Filters.h deleted file mode 100644 index f764b0b..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/Filters.h +++ /dev/null @@ -1,105 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2020-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -#ifndef CVCORE_FILTERS_H -#define CVCORE_FILTERS_H - -#include "cv/core/Tensor.h" -#include "cv/core/MathTypes.h" - -#include - -namespace cvcore { namespace tensor_ops { - -/** - * Box type filtering for three channel HWC format uint_8 type Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param maskSize Size of mask which determines number of pixels to be averaged. - * @param anchor Offset of mask relative to current pixel index. - * {0, 0} mask aligns with starting pixel. - * {mask size/2, mask size/2} mask aligns with center pixel index. - * @param stream specified cuda stream. - */ - -void BoxFilter(Tensor &dst, const Tensor &src, const Vector2i &maskSize, - const Vector2i &anchor, cudaStream_t stream = 0); -/** - * Box type filtering for three channel HWC format uint_16 type Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param maskSize Size of mask which determines number of pixels to be averaged. - * @param anchor Offset of mask relative to current pixel index. - * {0, 0} mask aligns with starting pixel. - * {mask size/2, mask size/2} mask aligns with center pixel index. - * @param stream specified cuda stream. - */ -void BoxFilter(Tensor &dst, const Tensor &src, const Vector2i &maskSize, - const Vector2i &anchor, cudaStream_t stream = 0); -/** - * Box type filtering for three channel HWC format float type Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param maskSize Size of mask which determines number of pixels to be averaged. - * @param anchor Offset of mask relative to current pixel index. - * {0, 0} mask aligns with starting pixel. - * {mask size/2, mask size/2} mask aligns with center pixel index. - * @param stream specified cuda stream. - */ -void BoxFilter(Tensor &dst, const Tensor &src, const Vector2i &maskSize, - const Vector2i &anchor, cudaStream_t stream = 0); - -/** - * Box type filtering for one channel HWC format uint_8 type Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param maskSize Size of mask which determines number of pixels to be averaged. - * @param anchor Offset of mask relative to current pixel index. - * {0, 0} mask aligns with starting pixel. - * {mask size/2, mask size/2} mask aligns with center pixel index. - * @param stream specified cuda stream. - */ -void BoxFilter(Tensor &dst, const Tensor &src, const Vector2i &maskSize, - const Vector2i &anchor, cudaStream_t stream = 0); -/** - * Box type filtering for one channel HWC format uint_16 type Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param maskSize Size of mask which determines number of pixels to be averaged. - * @param anchor Offset of mask relative to current pixel index. - * {0, 0} mask aligns with starting pixel. - * {mask size/2, mask size/2} mask aligns with center pixel index. - * @param stream specified cuda stream. - */ -void BoxFilter(Tensor &dst, const Tensor &src, const Vector2i &maskSize, - const Vector2i &anchor, cudaStream_t stream = 0); -/** - * Box type filtering for one channel HWC format float type Tensor. - * @param dst destination tensor. - * @param src source tensor. - * @param maskSize Size of mask which determines number of pixels to be averaged. - * @param anchor Offset of mask relative to current pixel index. - * {0, 0} mask aligns with starting pixel. - * {mask size/2, mask size/2} mask aligns with center pixel index. - * @param stream specified cuda stream. - */ -void BoxFilter(Tensor &dst, const Tensor &src, const Vector2i &maskSize, - const Vector2i &anchor, cudaStream_t stream = 0); - -}} // namespace cvcore::tensor_ops - -#endif // CVCORE_FILTERS_H diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/FusedOperations.cpp b/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/FusedOperations.cpp deleted file mode 100644 index 4ae0c3b..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/FusedOperations.cpp +++ /dev/null @@ -1,261 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2020-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -#include "NppUtils.h" - -#include "cv/tensor_ops/ImageUtils.h" - -#include "cv/core/Memory.h" - -#include -#include -#include - -namespace cvcore { namespace tensor_ops { - -template -struct ImageToNormalizedPlanarTensorOperator::ImageToNormalizedPlanarTensorOperatorImpl -{ - int m_width; - int m_height; - int m_depth; - std::unique_ptr> m_resizedTensor; - std::unique_ptr> m_normalizedTensor; - - template::type * = nullptr> - ImageToNormalizedPlanarTensorOperatorImpl(int width, int height) - : m_width(width) - , m_height(height) - , m_depth(1) - { - m_resizedTensor.reset(new Tensor(width, height, false)); - m_normalizedTensor.reset(new Tensor(width, height, false)); - } - - template::type * = nullptr> - ImageToNormalizedPlanarTensorOperatorImpl(int width, int height, int depth) - : m_width(width) - , m_height(height) - , m_depth(depth) - { - m_resizedTensor.reset(new Tensor(width, height, depth, false)); - m_normalizedTensor.reset(new Tensor(width, height, depth, false)); - } - - template::type * = nullptr> - void execute(Tensor &dst, const Tensor &src, const float scale[3], - const float offset[3], bool swapRB, bool keep_aspect_ratio, cudaStream_t stream) - { - // src and dst must be GPU tensors - assert(!src.isCPU() && !dst.isCPU()); - - // dst image width/height must match width/height of class - if ((dst.getWidth() != m_width) || (dst.getHeight() != m_height)) - { - throw std::runtime_error("invalid input width/height"); - } - - // first do the resizing - Resize(*m_resizedTensor, src, keep_aspect_ratio, INTERP_LINEAR, stream); - - // swap channels if needed - if (swapRB) - { - ConvertColorFormat(*m_resizedTensor, *m_resizedTensor, BGR2RGB, stream); - } - - // do the normalization - Normalize(*m_normalizedTensor, *m_resizedTensor, scale, offset, stream); - - // convert interleave to planar tensor - InterleavedToPlanar(dst, *m_normalizedTensor, stream); - } - - template::type * = nullptr> - void execute(Tensor &dst, const Tensor &src, const float scale[3], - const float offset[3], bool swapRB, bool keep_aspect_ratio, cudaStream_t stream) - { - // src and dst must be GPU tensors - assert(!src.isCPU() && !dst.isCPU()); - - // dst image width/height must match width/height of class - if ((dst.getWidth() != m_width) || (dst.getHeight() != m_height)) - { - throw std::runtime_error("invalid input width/height"); - } - - // dst image depth must be equal to src image depth and no bigger than m_depth - if ((dst.getDepth() != src.getDepth()) || (dst.getDepth() > m_depth)) - { - throw std::runtime_error("invalid input depth"); - } - - // wrap the batch tensor with non-owning tensor - Tensor resizedTensor(m_width, m_height, dst.getDepth(), m_resizedTensor->getData(), false); - Tensor normalizedTensor(m_width, m_height, dst.getDepth(), m_normalizedTensor->getData(), - false); - - // first do the resizing - Resize(resizedTensor, src, keep_aspect_ratio, INTERP_LINEAR, stream); - - // swap channels if needed - if (swapRB) - { - ConvertColorFormat(resizedTensor, resizedTensor, BGR2RGB, stream); - } - - // do the normalization - Normalize(normalizedTensor, resizedTensor, scale, offset, stream); - - // convert interleave to planar tensor - InterleavedToPlanar(dst, normalizedTensor, stream); - } - - template::type * = nullptr> - void execute(Tensor &dst, const Tensor &src, float scale, float offset, - bool keep_aspect_ratio, cudaStream_t stream) - { - // src and dst must be GPU tensors - assert(!src.isCPU() && !dst.isCPU()); - - // dst image width/height must match width/height of class - if ((dst.getWidth() != m_width) || (dst.getHeight() != m_height)) - { - throw std::runtime_error("invalid input width/height"); - } - - // first do the resizing - Resize(*m_resizedTensor, src, keep_aspect_ratio, INTERP_LINEAR, stream); - - // do the normalization and map to destination tensor directly - Tensor output(m_width, m_height, dst.getData(), false); - Normalize(output, *m_resizedTensor, scale, offset, stream); - } - - template::type * = nullptr> - void execute(Tensor &dst, const Tensor &src, float scale, float offset, - bool keep_aspect_ratio, cudaStream_t stream) - { - // src and dst must be GPU tensors - assert(!src.isCPU() && !dst.isCPU()); - - // dst image width/height must match width/height of class - if ((dst.getWidth() != m_width) || (dst.getHeight() != m_height)) - { - throw std::runtime_error("invalid input width/height"); - } - - // dst image depth must be equal to src image depth and no bigger than m_depth - if ((dst.getDepth() != src.getDepth()) || (dst.getDepth() > m_depth)) - { - throw std::runtime_error("invalid input depth"); - } - - // wrap the batch tensor with non-owning tensor - Tensor resizedTensor(m_width, m_height, dst.getDepth(), m_resizedTensor->getData(), false); - - // first do the resizing - Resize(resizedTensor, src, keep_aspect_ratio, INTERP_LINEAR, stream); - - // do the normalization and map to destination tensor directly - Tensor output(m_width, m_height, dst.getDepth(), dst.getData(), false); - Normalize(output, resizedTensor, scale, offset, stream); - } -}; - -template -template::type *> -ImageToNormalizedPlanarTensorOperator::ImageToNormalizedPlanarTensorOperator(int width, - int height) - : m_pImpl(new ImageToNormalizedPlanarTensorOperatorImpl(width, height)) -{ - static_assert(TL_IN == HWC && TL_OUT == CHW, "Tensor Layout is different"); - static_assert(CC == C1 || CC == C3, "Channel count is different"); -} - -template -template::type *> -ImageToNormalizedPlanarTensorOperator::ImageToNormalizedPlanarTensorOperator(int width, - int height, - int depth) - : m_pImpl(new ImageToNormalizedPlanarTensorOperatorImpl(width, height, depth)) -{ - static_assert(TL_IN == NHWC && TL_OUT == NCHW, "Tensor Layout is different"); - static_assert(CC == C1 || CC == C3, "Channel count is different"); -} - -template -ImageToNormalizedPlanarTensorOperator::~ImageToNormalizedPlanarTensorOperator() -{ -} - -template -template::type *> -void ImageToNormalizedPlanarTensorOperator::operator()( - Tensor &dst, const Tensor &src, const float scale[3], const float offset[3], - bool swapRB, bool keep_aspect_ratio, cudaStream_t stream) -{ - m_pImpl->execute(dst, src, scale, offset, swapRB, keep_aspect_ratio, stream); -} - -template -template::type *> -void ImageToNormalizedPlanarTensorOperator::operator()(Tensor &dst, - const Tensor &src, - float scale, float offset, - bool keep_aspect_ratio, - cudaStream_t stream) -{ - m_pImpl->execute(dst, src, scale, offset, keep_aspect_ratio, stream); -} - -// explicit instantiations -template class ImageToNormalizedPlanarTensorOperator; -template void ImageToNormalizedPlanarTensorOperator::operator()(Tensor &, - const Tensor &, - const float [], const float [], - bool, bool, cudaStream_t); -template ImageToNormalizedPlanarTensorOperator::ImageToNormalizedPlanarTensorOperator(int, int); - -template class ImageToNormalizedPlanarTensorOperator; -template void ImageToNormalizedPlanarTensorOperator::operator()(Tensor &, - const Tensor &, - float, float, bool, cudaStream_t); -template ImageToNormalizedPlanarTensorOperator::ImageToNormalizedPlanarTensorOperator(int, int); - -template class ImageToNormalizedPlanarTensorOperator; -template void ImageToNormalizedPlanarTensorOperator::operator()(Tensor &, - const Tensor &, - const float [], const float [], - bool, bool, cudaStream_t); -template ImageToNormalizedPlanarTensorOperator::ImageToNormalizedPlanarTensorOperator(int, - int, - int); - -template class ImageToNormalizedPlanarTensorOperator; -template void ImageToNormalizedPlanarTensorOperator::operator()(Tensor &, - const Tensor &, - float, float, bool, - cudaStream_t); -template ImageToNormalizedPlanarTensorOperator::ImageToNormalizedPlanarTensorOperator(int, - int, - int); -}} // namespace cvcore::tensor_ops diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/GeometryTransforms.cpp b/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/GeometryTransforms.cpp deleted file mode 100644 index 238eac1..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/GeometryTransforms.cpp +++ /dev/null @@ -1,754 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2020-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -#include "NppUtils.h" - -#include "cv/tensor_ops/ImageUtils.h" - -#include "cv/core/Memory.h" - -#include -#include - -#include -#include -#include - -namespace cvcore { namespace tensor_ops { - -namespace { - -static NppiInterpolationMode GetNppiInterpolationMode(InterpolationType type) -{ - if (type == INTERP_NEAREST) - { - return NPPI_INTER_NN; - } - else if (type == INTERP_LINEAR) - { - return NPPI_INTER_LINEAR; - } - else if (type == INTERP_CUBIC_BSPLINE) - { - return NPPI_INTER_CUBIC2P_BSPLINE; - } - else if (type == INTERP_CUBIC_CATMULLROM) - { - return NPPI_INTER_CUBIC2P_CATMULLROM; - } - else - { - throw std::runtime_error("invalid resizing interpolation mode"); - } -} - -static BBox GetScaledROI(int srcW, int srcH, int dstW, int dstH) -{ - if (srcW * dstH >= dstW * srcH) - { - int bboxH = static_cast((static_cast(srcH) / srcW) * dstW); - int offsetH = (dstH - bboxH) / 2; - return {0, offsetH, dstW, offsetH + bboxH}; - } - else - { - int bboxW = static_cast((static_cast(srcW) / srcH) * dstH); - int offsetW = (dstW - bboxW) / 2; - return {offsetW, 0, offsetW + bboxW, dstH}; - } -} - -static void AssertValidROI(const BBox &roi, int width, int height) -{ - assert(roi.xmin >= 0 && roi.xmin < roi.xmax); - assert(roi.ymin >= 0 && roi.ymin < roi.ymax); - assert(roi.ymax <= height); - assert(roi.xmax <= width); -} - -template -void FillBufferC1U8Impl(Tensor &dst, const Npp8u value, cudaStream_t stream) -{ - assert(!dst.isCPU()); - NppStatus status = nppiSet_8u_C1R_Ctx(value, static_cast(dst.getData()), - dst.getStride(TensorDimension::HEIGHT) * sizeof(Npp8u), - {int(dst.getWidth()), int(dst.getHeight())}, GetNppStreamContext(stream)); - assert(status == NPP_SUCCESS); -} - -template -void FillBufferC1U16Impl(Tensor &dst, const Npp16u value, cudaStream_t stream) -{ - assert(!dst.isCPU()); - NppStatus status = nppiSet_16u_C1R_Ctx(value, static_cast(dst.getData()), - dst.getStride(TensorDimension::HEIGHT) * sizeof(Npp16u), - {int(dst.getWidth()), int(dst.getHeight())}, GetNppStreamContext(stream)); - assert(status == NPP_SUCCESS); -} - -template -void FillBufferC1F32Impl(Tensor &dst, const Npp32f value, cudaStream_t stream) -{ - assert(!dst.isCPU()); - NppStatus status = nppiSet_32f_C1R_Ctx(value, static_cast(dst.getData()), - dst.getStride(TensorDimension::HEIGHT) * sizeof(Npp32f), - {int(dst.getWidth()), int(dst.getHeight())}, GetNppStreamContext(stream)); - assert(status == NPP_SUCCESS); -} - -static void FillBuffer(Tensor &dst, const Npp8u value, cudaStream_t stream = 0) -{ - FillBufferC1U8Impl(dst, value, stream); -} - -static void FillBuffer(Tensor &dst, const Npp16u value, cudaStream_t stream = 0) -{ - FillBufferC1U16Impl(dst, value, stream); -} - -static void FillBuffer(Tensor &dst, const Npp32f value, cudaStream_t stream = 0) -{ - FillBufferC1F32Impl(dst, value, stream); -} - -static void FillBuffer(Tensor &dst, const Npp8u value, cudaStream_t stream = 0) -{ - FillBufferC1U8Impl(dst, value, stream); -} - -static void FillBuffer(Tensor &dst, const Npp16u value, cudaStream_t stream = 0) -{ - FillBufferC1U16Impl(dst, value, stream); -} - -static void FillBuffer(Tensor &dst, const Npp32f value, cudaStream_t stream = 0) -{ - FillBufferC1F32Impl(dst, value, stream); -} - -static void FillBuffer(Tensor &dst, const Npp8u value, cudaStream_t stream = 0) -{ - assert(!dst.isCPU()); - const Npp8u padding[3] = {value, value, value}; - NppStatus status = nppiSet_8u_C3R_Ctx(padding, static_cast(dst.getData()), - dst.getStride(TensorDimension::HEIGHT) * sizeof(Npp8u), - {int(dst.getWidth()), int(dst.getHeight())}, GetNppStreamContext(stream)); - assert(status == NPP_SUCCESS); -} - -static void FillBuffer(Tensor &dst, const Npp16u value, cudaStream_t stream = 0) -{ - assert(!dst.isCPU()); - const Npp16u padding[3] = {value, value, value}; - NppStatus status = nppiSet_16u_C3R_Ctx(padding, static_cast(dst.getData()), - dst.getStride(TensorDimension::HEIGHT) * sizeof(Npp16u), - {int(dst.getWidth()), int(dst.getHeight())}, GetNppStreamContext(stream)); - assert(status == NPP_SUCCESS); -} - -static void FillBuffer(Tensor &dst, const Npp32f value, cudaStream_t stream = 0) -{ - assert(!dst.isCPU()); - const Npp32f padding[3] = {value, value, value}; - NppStatus status = nppiSet_32f_C3R_Ctx(padding, static_cast(dst.getData()), - dst.getStride(TensorDimension::HEIGHT) * sizeof(Npp32f), - {int(dst.getWidth()), int(dst.getHeight())}, GetNppStreamContext(stream)); - assert(status == NPP_SUCCESS); -} - -template -void CropAndResizeC1U8Impl(Tensor &dst, const Tensor &src, const BBox &dstROI, - const BBox &srcROI, InterpolationType type, cudaStream_t stream) -{ - // src and dst must be GPU tensors - assert(!src.isCPU() && !dst.isCPU()); - AssertValidROI(dstROI, dst.getWidth(), dst.getHeight()); - AssertValidROI(srcROI, src.getWidth(), src.getHeight()); - - NppStatus status = nppiResizeSqrPixel_8u_C1R_Ctx( - static_cast(src.getData() + srcROI.ymin * src.getStride(TensorDimension::HEIGHT) + srcROI.xmin), - {srcROI.xmax - srcROI.xmin, srcROI.ymax - srcROI.ymin}, src.getStride(TensorDimension::HEIGHT) * sizeof(Npp8u), - {0, 0, srcROI.xmax - srcROI.xmin, srcROI.ymax - srcROI.ymin}, - static_cast(dst.getData() + dstROI.ymin * dst.getStride(TensorDimension::HEIGHT) + dstROI.xmin), - dst.getStride(TensorDimension::HEIGHT) * sizeof(Npp8u), - {0, 0, dstROI.xmax - dstROI.xmin, dstROI.ymax - dstROI.ymin}, - double(dstROI.xmax - dstROI.xmin) / double(srcROI.xmax - srcROI.xmin), - double(dstROI.ymax - dstROI.ymin) / double(srcROI.ymax - srcROI.ymin), 0.0, 0.0, GetNppiInterpolationMode(type), - GetNppStreamContext(stream)); - assert(status == NPP_SUCCESS); -} - -template -void CropAndResizeC1U16Impl(Tensor &dst, const Tensor &src, const BBox &dstROI, - const BBox &srcROI, InterpolationType type, cudaStream_t stream) -{ - // src and dst must be GPU tensors - assert(!src.isCPU() && !dst.isCPU()); - AssertValidROI(dstROI, dst.getWidth(), dst.getHeight()); - AssertValidROI(srcROI, src.getWidth(), src.getHeight()); - - NppStatus status = nppiResizeSqrPixel_16u_C1R_Ctx( - static_cast(src.getData() + srcROI.ymin * src.getStride(TensorDimension::HEIGHT) + srcROI.xmin), - {srcROI.xmax - srcROI.xmin, srcROI.ymax - srcROI.ymin}, src.getStride(TensorDimension::HEIGHT) * sizeof(Npp16u), - {0, 0, srcROI.xmax - srcROI.xmin, srcROI.ymax - srcROI.ymin}, - static_cast(dst.getData() + dstROI.ymin * dst.getStride(TensorDimension::HEIGHT) + dstROI.xmin), - dst.getStride(TensorDimension::HEIGHT) * sizeof(Npp16u), - {0, 0, dstROI.xmax - dstROI.xmin, dstROI.ymax - dstROI.ymin}, - double(dstROI.xmax - dstROI.xmin) / double(srcROI.xmax - srcROI.xmin), - double(dstROI.ymax - dstROI.ymin) / double(srcROI.ymax - srcROI.ymin), 0.0, 0.0, GetNppiInterpolationMode(type), - GetNppStreamContext(stream)); - assert(status == NPP_SUCCESS); -} - -template -void CropAndResizeC1F32Impl(Tensor &dst, const Tensor &src, const BBox &dstROI, - const BBox &srcROI, InterpolationType type, cudaStream_t stream) -{ - // src and dst must be GPU tensors - assert(!src.isCPU() && !dst.isCPU()); - AssertValidROI(dstROI, dst.getWidth(), dst.getHeight()); - AssertValidROI(srcROI, src.getWidth(), src.getHeight()); - - NppStatus status = nppiResizeSqrPixel_32f_C1R_Ctx( - static_cast(src.getData() + srcROI.ymin * src.getStride(TensorDimension::HEIGHT) + srcROI.xmin), - {srcROI.xmax - srcROI.xmin, srcROI.ymax - srcROI.ymin}, src.getStride(TensorDimension::HEIGHT) * sizeof(Npp32f), - {0, 0, srcROI.xmax - srcROI.xmin, srcROI.ymax - srcROI.ymin}, - static_cast(dst.getData() + dstROI.ymin * dst.getStride(TensorDimension::HEIGHT) + dstROI.xmin), - dst.getStride(TensorDimension::HEIGHT) * sizeof(Npp32f), - {0, 0, dstROI.xmax - dstROI.xmin, dstROI.ymax - dstROI.ymin}, - double(dstROI.xmax - dstROI.xmin) / double(srcROI.xmax - srcROI.xmin), - double(dstROI.ymax - dstROI.ymin) / double(srcROI.ymax - srcROI.ymin), 0.0, 0.0, GetNppiInterpolationMode(type), - GetNppStreamContext(stream)); - assert(status == NPP_SUCCESS); -} - -template -void ResizeImpl(Tensor &dst, const Tensor &src, bool keep_aspect_ratio, InterpolationType type, - cudaStream_t stream) -{ - const BBox dstROI = keep_aspect_ratio - ? GetScaledROI(src.getWidth(), src.getHeight(), dst.getWidth(), dst.getHeight()) - : BBox{0, 0, int(dst.getWidth()), int(dst.getHeight())}; - if (keep_aspect_ratio) - { - FillBuffer(dst, 0, stream); - } - CropAndResize(dst, src, dstROI, {0, 0, int(src.getWidth()), int(src.getHeight())}, type, stream); -} - -template -void CropC1U8Impl(Tensor &dst, const Tensor &src, const BBox &srcROI, cudaStream_t stream) -{ - // src and dst must be GPU tensors - assert(!src.isCPU() && !dst.isCPU()); - AssertValidROI(srcROI, src.getWidth(), src.getHeight()); - assert(srcROI.xmax - srcROI.xmin == dst.getWidth() && srcROI.ymax - srcROI.ymin == dst.getHeight()); - - NppStatus status = nppiCopy_8u_C1R_Ctx( - static_cast(src.getData() + srcROI.ymin * src.getStride(TensorDimension::HEIGHT) + srcROI.xmin), - src.getStride(TensorDimension::HEIGHT) * sizeof(Npp8u), static_cast(dst.getData()), - dst.getStride(TensorDimension::HEIGHT) * sizeof(Npp8u), {srcROI.xmax - srcROI.xmin, srcROI.ymax - srcROI.ymin}, - GetNppStreamContext(stream)); - - assert(status == NPP_SUCCESS); -} - -template -void CropC1U16Impl(Tensor &dst, const Tensor &src, const BBox &srcROI, cudaStream_t stream) -{ - // src and dst must be GPU tensors - assert(!src.isCPU() && !dst.isCPU()); - AssertValidROI(srcROI, src.getWidth(), src.getHeight()); - assert(srcROI.xmax - srcROI.xmin == dst.getWidth() && srcROI.ymax - srcROI.ymin == dst.getHeight()); - - NppStatus status = nppiCopy_16u_C1R_Ctx( - static_cast(src.getData() + srcROI.ymin * src.getStride(TensorDimension::HEIGHT) + srcROI.xmin), - src.getStride(TensorDimension::HEIGHT) * sizeof(Npp16u), static_cast(dst.getData()), - dst.getStride(TensorDimension::HEIGHT) * sizeof(Npp16u), {srcROI.xmax - srcROI.xmin, srcROI.ymax - srcROI.ymin}, - GetNppStreamContext(stream)); - - assert(status == NPP_SUCCESS); -} - -template -void CropC1F32Impl(Tensor &dst, const Tensor &src, const BBox &srcROI, cudaStream_t stream) -{ - // src and dst must be GPU tensors - assert(!src.isCPU() && !dst.isCPU()); - AssertValidROI(srcROI, src.getWidth(), src.getHeight()); - assert(srcROI.xmax - srcROI.xmin == dst.getWidth() && srcROI.ymax - srcROI.ymin == dst.getHeight()); - - NppStatus status = nppiCopy_32f_C1R_Ctx( - static_cast(src.getData() + srcROI.ymin * src.getStride(TensorDimension::HEIGHT) + srcROI.xmin), - src.getStride(TensorDimension::HEIGHT) * sizeof(Npp32f), static_cast(dst.getData()), - dst.getStride(TensorDimension::HEIGHT) * sizeof(Npp32f), {srcROI.xmax - srcROI.xmin, srcROI.ymax - srcROI.ymin}, - GetNppStreamContext(stream)); - - assert(status == NPP_SUCCESS); -} - -template -void ResizeBatch(Tensor &dst, Tensor &src, bool keep_aspect_ratio, InterpolationType type, - cudaStream_t stream) -{ - // src and dst must be GPU tensors - assert(!src.isCPU() && !dst.isCPU()); - assert(src.getDepth() == dst.getDepth()); - - for (int i = 0; i < src.getDepth(); i++) - { - size_t offsetSrc = i * src.getStride(TensorDimension::DEPTH); - size_t offsetDst = i * dst.getStride(TensorDimension::DEPTH); - Tensor srcTmp(src.getWidth(), src.getHeight(), - src.getStride(TensorDimension::HEIGHT) * GetChannelSize(CT), - src.getData() + offsetSrc, false); - Tensor dstTmp(dst.getWidth(), dst.getHeight(), - dst.getStride(TensorDimension::HEIGHT) * GetChannelSize(CT), - dst.getData() + offsetDst, false); - Resize(dstTmp, srcTmp, keep_aspect_ratio, type, stream); - } -} - -} // anonymous namespace - -void Resize(Tensor &dst, const Tensor &src, bool keep_aspect_ratio, InterpolationType type, - cudaStream_t stream) -{ - ResizeImpl(dst, src, keep_aspect_ratio, type, stream); -} - -void Resize(Tensor &dst, const Tensor &src, bool keep_aspect_ratio, InterpolationType type, - cudaStream_t stream) -{ - ResizeBatch(dst, const_cast &>(src), keep_aspect_ratio, type, stream); -} - -void CropAndResize(Tensor &dst, const Tensor &src, const BBox &dstROI, const BBox &srcROI, - InterpolationType type, cudaStream_t stream) -{ - CropAndResizeC1U8Impl(dst, src, dstROI, srcROI, type, stream); -} - -void CropAndResize(Tensor &dst, const Tensor &src, const BBox &srcROI, InterpolationType type, - cudaStream_t stream) -{ - CropAndResize(dst, src, {0, 0, int(dst.getWidth()), int(dst.getHeight())}, srcROI, type, stream); -} - -void Resize(Tensor &dst, const Tensor &src, bool keep_aspect_ratio, InterpolationType type, - cudaStream_t stream) -{ - ResizeImpl(dst, src, keep_aspect_ratio, type, stream); -} - -void Resize(Tensor &dst, const Tensor &src, bool keep_aspect_ratio, - InterpolationType type, cudaStream_t stream) -{ - ResizeBatch(dst, const_cast &>(src), keep_aspect_ratio, type, stream); -} - -void CropAndResize(Tensor &dst, const Tensor &src, const BBox &dstROI, const BBox &srcROI, - InterpolationType type, cudaStream_t stream) -{ - CropAndResizeC1U16Impl(dst, src, dstROI, srcROI, type, stream); -} - -void CropAndResize(Tensor &dst, const Tensor &src, const BBox &srcROI, - InterpolationType type, cudaStream_t stream) -{ - CropAndResize(dst, src, {0, 0, int(dst.getWidth()), int(dst.getHeight())}, srcROI, type, stream); -} - -void Resize(Tensor &dst, const Tensor &src, bool keep_aspect_ratio, InterpolationType type, - cudaStream_t stream) -{ - ResizeImpl(dst, src, keep_aspect_ratio, type, stream); -} - -void Resize(Tensor &dst, const Tensor &src, bool keep_aspect_ratio, - InterpolationType type, cudaStream_t stream) -{ - ResizeBatch(dst, const_cast &>(src), keep_aspect_ratio, type, stream); -} - -void CropAndResize(Tensor &dst, const Tensor &src, const BBox &dstROI, const BBox &srcROI, - InterpolationType type, cudaStream_t stream) -{ - CropAndResizeC1F32Impl(dst, src, dstROI, srcROI, type, stream); -} - -void CropAndResize(Tensor &dst, const Tensor &src, const BBox &srcROI, - InterpolationType type, cudaStream_t stream) -{ - CropAndResize(dst, src, {0, 0, int(dst.getWidth()), int(dst.getHeight())}, srcROI, type, stream); -} - -void Resize(Tensor &dst, const Tensor &src, bool keep_aspect_ratio, InterpolationType type, - cudaStream_t stream) -{ - ResizeImpl(dst, src, keep_aspect_ratio, type, stream); -} - -void CropAndResize(Tensor &dst, const Tensor &src, const BBox &dstROI, const BBox &srcROI, - InterpolationType type, cudaStream_t stream) -{ - CropAndResizeC1U8Impl(dst, src, dstROI, srcROI, type, stream); -} - -void CropAndResize(Tensor &dst, const Tensor &src, const BBox &srcROI, InterpolationType type, - cudaStream_t stream) -{ - CropAndResize(dst, src, {0, 0, int(dst.getWidth()), int(dst.getHeight())}, srcROI, type, stream); -} - -void Resize(Tensor &dst, const Tensor &src, bool keep_aspect_ratio, InterpolationType type, - cudaStream_t stream) -{ - ResizeImpl(dst, src, keep_aspect_ratio, type, stream); -} - -void CropAndResize(Tensor &dst, const Tensor &src, const BBox &dstROI, const BBox &srcROI, - InterpolationType type, cudaStream_t stream) -{ - CropAndResizeC1U16Impl(dst, src, dstROI, srcROI, type, stream); -} - -void CropAndResize(Tensor &dst, const Tensor &src, const BBox &srcROI, - InterpolationType type, cudaStream_t stream) -{ - CropAndResize(dst, src, {0, 0, int(dst.getWidth()), int(dst.getHeight())}, srcROI, type, stream); -} - -void Resize(Tensor &dst, const Tensor &src, bool keep_aspect_ratio, InterpolationType type, - cudaStream_t stream) -{ - ResizeImpl(dst, src, keep_aspect_ratio, type, stream); -} - -void CropAndResize(Tensor &dst, const Tensor &src, const BBox &dstROI, const BBox &srcROI, - InterpolationType type, cudaStream_t stream) -{ - CropAndResizeC1F32Impl(dst, src, dstROI, srcROI, type, stream); -} - -void CropAndResize(Tensor &dst, const Tensor &src, const BBox &srcROI, - InterpolationType type, cudaStream_t stream) -{ - CropAndResize(dst, src, {0, 0, int(dst.getWidth()), int(dst.getHeight())}, srcROI, type, stream); -} - -void Resize(Tensor &dst, const Tensor &src, bool keep_aspect_ratio, InterpolationType type, - cudaStream_t stream) -{ - ResizeImpl(dst, src, keep_aspect_ratio, type, stream); -} - -void Resize(Tensor &dst, const Tensor &src, bool keep_aspect_ratio, InterpolationType type, - cudaStream_t stream) -{ - ResizeBatch(dst, const_cast &>(src), keep_aspect_ratio, type, stream); -} - -void CropAndResize(Tensor &dst, const Tensor &src, const BBox &dstROI, const BBox &srcROI, - InterpolationType type, cudaStream_t stream) -{ - // src and dst must be GPU tensors - assert(!src.isCPU() && !dst.isCPU()); - AssertValidROI(dstROI, dst.getWidth(), dst.getHeight()); - AssertValidROI(srcROI, src.getWidth(), src.getHeight()); - - NppStatus status = nppiResizeSqrPixel_8u_C3R_Ctx( - static_cast(src.getData() + srcROI.ymin * src.getStride(TensorDimension::HEIGHT) + - srcROI.xmin * src.getChannelCount()), - {srcROI.xmax - srcROI.xmin, srcROI.ymax - srcROI.ymin}, src.getStride(TensorDimension::HEIGHT) * sizeof(Npp8u), - {0, 0, srcROI.xmax - srcROI.xmin, srcROI.ymax - srcROI.ymin}, - static_cast(dst.getData() + dstROI.ymin * dst.getStride(TensorDimension::HEIGHT) + - dstROI.xmin * dst.getChannelCount()), - dst.getStride(TensorDimension::HEIGHT) * sizeof(Npp8u), - {0, 0, dstROI.xmax - dstROI.xmin, dstROI.ymax - dstROI.ymin}, - double(dstROI.xmax - dstROI.xmin) / double(srcROI.xmax - srcROI.xmin), - double(dstROI.ymax - dstROI.ymin) / double(srcROI.ymax - srcROI.ymin), 0.0, 0.0, GetNppiInterpolationMode(type), - GetNppStreamContext(stream)); - assert(status == NPP_SUCCESS); -} - -void CropAndResize(Tensor &dst, const Tensor &src, const BBox &srcROI, InterpolationType type, - cudaStream_t stream) -{ - CropAndResize(dst, src, {0, 0, int(dst.getWidth()), int(dst.getHeight())}, srcROI, type, stream); -} - -void Resize(Tensor &dst, const Tensor &src, bool keep_aspect_ratio, InterpolationType type, - cudaStream_t stream) -{ - ResizeImpl(dst, src, keep_aspect_ratio, type, stream); -} - -void Resize(Tensor &dst, const Tensor &src, bool keep_aspect_ratio, - InterpolationType type, cudaStream_t stream) -{ - ResizeBatch(dst, const_cast &>(src), keep_aspect_ratio, type, stream); -} - -void CropAndResize(Tensor &dst, const Tensor &src, const BBox &dstROI, const BBox &srcROI, - InterpolationType type, cudaStream_t stream) -{ - // src and dst must be GPU tensors - assert(!src.isCPU() && !dst.isCPU()); - AssertValidROI(dstROI, dst.getWidth(), dst.getHeight()); - AssertValidROI(srcROI, src.getWidth(), src.getHeight()); - - NppStatus status = nppiResizeSqrPixel_16u_C3R_Ctx( - static_cast(src.getData() + srcROI.ymin * src.getStride(TensorDimension::HEIGHT) + - srcROI.xmin * src.getChannelCount()), - {srcROI.xmax - srcROI.xmin, srcROI.ymax - srcROI.ymin}, src.getStride(TensorDimension::HEIGHT) * sizeof(Npp16u), - {0, 0, srcROI.xmax - srcROI.xmin, srcROI.ymax - srcROI.ymin}, - static_cast(dst.getData() + dstROI.ymin * dst.getStride(TensorDimension::HEIGHT) + - dstROI.xmin * dst.getChannelCount()), - dst.getStride(TensorDimension::HEIGHT) * sizeof(Npp16u), - {0, 0, dstROI.xmax - dstROI.xmin, dstROI.ymax - dstROI.ymin}, - double(dstROI.xmax - dstROI.xmin) / double(srcROI.xmax - srcROI.xmin), - double(dstROI.ymax - dstROI.ymin) / double(srcROI.ymax - srcROI.ymin), 0.0, 0.0, GetNppiInterpolationMode(type), - GetNppStreamContext(stream)); - assert(status == NPP_SUCCESS); -} - -void CropAndResize(Tensor &dst, const Tensor &src, const BBox &srcROI, - InterpolationType type, cudaStream_t stream) -{ - CropAndResize(dst, src, {0, 0, int(dst.getWidth()), int(dst.getHeight())}, srcROI, type, stream); -} - -void Resize(Tensor &dst, const Tensor &src, bool keep_aspect_ratio, InterpolationType type, - cudaStream_t stream) -{ - ResizeImpl(dst, src, keep_aspect_ratio, type, stream); -} - -void Resize(Tensor &dst, const Tensor &src, bool keep_aspect_ratio, - InterpolationType type, cudaStream_t stream) -{ - ResizeBatch(dst, const_cast &>(src), keep_aspect_ratio, type, stream); -} - -void CropAndResize(Tensor &dst, const Tensor &src, const BBox &dstROI, const BBox &srcROI, - InterpolationType type, cudaStream_t stream) -{ - // src and dst must be GPU tensors - assert(!src.isCPU() && !dst.isCPU()); - AssertValidROI(dstROI, dst.getWidth(), dst.getHeight()); - AssertValidROI(srcROI, src.getWidth(), src.getHeight()); - - NppStatus status = nppiResizeSqrPixel_32f_C3R_Ctx( - static_cast(src.getData() + srcROI.ymin * src.getStride(TensorDimension::HEIGHT) + - srcROI.xmin * src.getChannelCount()), - {srcROI.xmax - srcROI.xmin, srcROI.ymax - srcROI.ymin}, src.getStride(TensorDimension::HEIGHT) * sizeof(Npp32f), - {0, 0, srcROI.xmax - srcROI.xmin, srcROI.ymax - srcROI.ymin}, - static_cast(dst.getData() + dstROI.ymin * dst.getStride(TensorDimension::HEIGHT) + - dstROI.xmin * dst.getChannelCount()), - dst.getStride(TensorDimension::HEIGHT) * sizeof(Npp32f), - {0, 0, dstROI.xmax - dstROI.xmin, dstROI.ymax - dstROI.ymin}, - double(dstROI.xmax - dstROI.xmin) / double(srcROI.xmax - srcROI.xmin), - double(dstROI.ymax - dstROI.ymin) / double(srcROI.ymax - srcROI.ymin), 0.0, 0.0, GetNppiInterpolationMode(type), - GetNppStreamContext(stream)); - assert(status == NPP_SUCCESS); -} - -void CropAndResize(Tensor &dst, const Tensor &src, const BBox &srcROI, - InterpolationType type, cudaStream_t stream) -{ - CropAndResize(dst, src, {0, 0, int(dst.getWidth()), int(dst.getHeight())}, srcROI, type, stream); -} - -void Crop(Tensor &dst, const Tensor &src, const BBox &srcROI, cudaStream_t stream) -{ - CropC1U8Impl(dst, src, srcROI, stream); -} - -void Crop(Tensor &dst, const Tensor &src, const BBox &srcROI, cudaStream_t stream) -{ - CropC1U16Impl(dst, src, srcROI, stream); -} - -void Crop(Tensor &dst, const Tensor &src, const BBox &srcROI, cudaStream_t stream) -{ - CropC1F32Impl(dst, src, srcROI, stream); -} - -void Crop(Tensor &dst, const Tensor &src, const BBox &srcROI, cudaStream_t stream) -{ - CropC1U8Impl(dst, src, srcROI, stream); -} - -void Crop(Tensor &dst, const Tensor &src, const BBox &srcROI, cudaStream_t stream) -{ - CropC1U16Impl(dst, src, srcROI, stream); -} - -void Crop(Tensor &dst, const Tensor &src, const BBox &srcROI, cudaStream_t stream) -{ - CropC1F32Impl(dst, src, srcROI, stream); -} - -void Crop(Tensor &dst, const Tensor &src, const BBox &srcROI, cudaStream_t stream) -{ - // src and dst must be GPU tensors - assert(!src.isCPU() && !dst.isCPU()); - AssertValidROI(srcROI, src.getWidth(), src.getHeight()); - assert(srcROI.xmax - srcROI.xmin == dst.getWidth() && srcROI.ymax - srcROI.ymin == dst.getHeight()); - - NppStatus status = nppiCopy_8u_C3R_Ctx( - static_cast(src.getData() + srcROI.ymin * src.getStride(TensorDimension::HEIGHT) + - srcROI.xmin * src.getChannelCount()), - src.getStride(TensorDimension::HEIGHT) * sizeof(Npp8u), static_cast(dst.getData()), - dst.getStride(TensorDimension::HEIGHT) * sizeof(Npp8u), {srcROI.xmax - srcROI.xmin, srcROI.ymax - srcROI.ymin}, - GetNppStreamContext(stream)); - - assert(status == NPP_SUCCESS); -} - -void Crop(Tensor &dst, const Tensor &src, const BBox &srcROI, cudaStream_t stream) -{ - // src and dst must be GPU tensors - assert(!src.isCPU() && !dst.isCPU()); - AssertValidROI(srcROI, src.getWidth(), src.getHeight()); - assert(srcROI.xmax - srcROI.xmin == dst.getWidth() && srcROI.ymax - srcROI.ymin == dst.getHeight()); - - NppStatus status = nppiCopy_16u_C3R_Ctx( - static_cast(src.getData() + srcROI.ymin * src.getStride(TensorDimension::HEIGHT) + - srcROI.xmin * src.getChannelCount()), - src.getStride(TensorDimension::HEIGHT) * sizeof(Npp16u), static_cast(dst.getData()), - dst.getStride(TensorDimension::HEIGHT) * sizeof(Npp16u), {srcROI.xmax - srcROI.xmin, srcROI.ymax - srcROI.ymin}, - GetNppStreamContext(stream)); - - assert(status == NPP_SUCCESS); -} - -void Crop(Tensor &dst, const Tensor &src, const BBox &srcROI, cudaStream_t stream) -{ - // src and dst must be GPU tensors - assert(!src.isCPU() && !dst.isCPU()); - AssertValidROI(srcROI, src.getWidth(), src.getHeight()); - assert(srcROI.xmax - srcROI.xmin == dst.getWidth() && srcROI.ymax - srcROI.ymin == dst.getHeight()); - - NppStatus status = nppiCopy_32f_C3R_Ctx( - static_cast(src.getData() + srcROI.ymin * src.getStride(TensorDimension::HEIGHT) + - srcROI.xmin * src.getChannelCount()), - src.getStride(TensorDimension::HEIGHT) * sizeof(Npp32f), static_cast(dst.getData()), - dst.getStride(TensorDimension::HEIGHT) * sizeof(Npp32f), {srcROI.xmax - srcROI.xmin, srcROI.ymax - srcROI.ymin}, - GetNppStreamContext(stream)); - - assert(status == NPP_SUCCESS); -} - -void WarpPerspective(Tensor &dst, const Tensor &src, const double coeffs[3][3], - InterpolationType type, cudaStream_t stream) -{ - // src and dst must be GPU tensors - assert(!src.isCPU() && !dst.isCPU()); - - NppStatus status = nppiWarpPerspective_8u_C1R_Ctx( - static_cast(src.getData()), {int(src.getWidth()), int(src.getHeight())}, - src.getStride(TensorDimension::HEIGHT) * sizeof(Npp8u), {0, 0, int(src.getWidth()), int(src.getHeight())}, - static_cast(dst.getData()), dst.getStride(TensorDimension::HEIGHT) * sizeof(Npp8u), - {0, 0, int(dst.getWidth()), int(dst.getHeight())}, coeffs, GetNppiInterpolationMode(type), - GetNppStreamContext(stream)); - - assert(status == NPP_SUCCESS); -} - -void WarpPerspective(Tensor &dst, const Tensor &src, const double coeffs[3][3], - InterpolationType type, cudaStream_t stream) -{ - // src and dst must be GPU tensors - assert(!src.isCPU() && !dst.isCPU()); - - NppStatus status = nppiWarpPerspective_16u_C1R_Ctx( - static_cast(src.getData()), {int(src.getWidth()), int(src.getHeight())}, - src.getStride(TensorDimension::HEIGHT) * sizeof(Npp16u), {0, 0, int(src.getWidth()), int(src.getHeight())}, - static_cast(dst.getData()), dst.getStride(TensorDimension::HEIGHT) * sizeof(Npp16u), - {0, 0, int(dst.getWidth()), int(dst.getHeight())}, coeffs, GetNppiInterpolationMode(type), - GetNppStreamContext(stream)); - - assert(status == NPP_SUCCESS); -} - -void WarpPerspective(Tensor &dst, const Tensor &src, const double coeffs[3][3], - InterpolationType type, cudaStream_t stream) -{ - // src and dst must be GPU tensors - assert(!src.isCPU() && !dst.isCPU()); - - NppStatus status = nppiWarpPerspective_32f_C1R_Ctx( - static_cast(src.getData()), {int(src.getWidth()), int(src.getHeight())}, - src.getStride(TensorDimension::HEIGHT) * sizeof(Npp32f), {0, 0, int(src.getWidth()), int(src.getHeight())}, - static_cast(dst.getData()), dst.getStride(TensorDimension::HEIGHT) * sizeof(Npp32f), - {0, 0, int(dst.getWidth()), int(dst.getHeight())}, coeffs, GetNppiInterpolationMode(type), - GetNppStreamContext(stream)); - - assert(status == NPP_SUCCESS); -} - -void WarpPerspective(Tensor &dst, const Tensor &src, const double coeffs[3][3], - InterpolationType type, cudaStream_t stream) -{ - // src and dst must be GPU tensors - assert(!src.isCPU() && !dst.isCPU()); - - NppStatus status = nppiWarpPerspective_8u_C3R_Ctx( - static_cast(src.getData()), {int(src.getWidth()), int(src.getHeight())}, - src.getStride(TensorDimension::HEIGHT) * sizeof(Npp8u), {0, 0, int(src.getWidth()), int(src.getHeight())}, - static_cast(dst.getData()), dst.getStride(TensorDimension::HEIGHT) * sizeof(Npp8u), - {0, 0, int(dst.getWidth()), int(dst.getHeight())}, coeffs, GetNppiInterpolationMode(type), - GetNppStreamContext(stream)); - - assert(status == NPP_SUCCESS); -} - -void WarpPerspective(Tensor &dst, const Tensor &src, const double coeffs[3][3], - InterpolationType type, cudaStream_t stream) -{ - // src and dst must be GPU tensors - assert(!src.isCPU() && !dst.isCPU()); - - NppStatus status = nppiWarpPerspective_16u_C3R_Ctx( - static_cast(src.getData()), {int(src.getWidth()), int(src.getHeight())}, - src.getStride(TensorDimension::HEIGHT) * sizeof(Npp16u), {0, 0, int(src.getWidth()), int(src.getHeight())}, - static_cast(dst.getData()), dst.getStride(TensorDimension::HEIGHT) * sizeof(Npp16u), - {0, 0, int(dst.getWidth()), int(dst.getHeight())}, coeffs, GetNppiInterpolationMode(type), - GetNppStreamContext(stream)); - - assert(status == NPP_SUCCESS); -} - -void WarpPerspective(Tensor &dst, const Tensor &src, const double coeffs[3][3], - InterpolationType type, cudaStream_t stream) -{ - // src and dst must be GPU tensors - assert(!src.isCPU() && !dst.isCPU()); - - NppStatus status = nppiWarpPerspective_32f_C3R_Ctx( - static_cast(src.getData()), {int(src.getWidth()), int(src.getHeight())}, - src.getStride(TensorDimension::HEIGHT) * sizeof(Npp32f), {0, 0, int(src.getWidth()), int(src.getHeight())}, - static_cast(dst.getData()), dst.getStride(TensorDimension::HEIGHT) * sizeof(Npp32f), - {0, 0, int(dst.getWidth()), int(dst.getHeight())}, coeffs, GetNppiInterpolationMode(type), - GetNppStreamContext(stream)); - - assert(status == NPP_SUCCESS); -} - -}} // namespace cvcore::tensor_ops \ No newline at end of file diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/NppUtils.cpp b/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/NppUtils.cpp deleted file mode 100644 index 4c7cf7f..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/NppUtils.cpp +++ /dev/null @@ -1,116 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2020-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -#include "NppUtils.h" - -#include -#include -#include -#include - -namespace cvcore { namespace tensor_ops { - -constexpr size_t CACHE_SIZE = 20; -static size_t timestamp = 0; -std::mutex lock; - -namespace { - -// This function involves GPU query and can be really slow -void SetupNppStreamContext(NppStreamContext &context, cudaStream_t stream) -{ - context.hStream = stream; - cudaError_t error = cudaGetDevice(&context.nCudaDeviceId); - if (error != cudaSuccess) - { - throw std::runtime_error("no devices supporting CUDA"); - } - error = cudaStreamGetFlags(context.hStream, &context.nStreamFlags); - if (error != cudaSuccess) - { - throw std::runtime_error("failed to get cuda stream flags"); - } - - cudaDeviceProp deviceProp; - error = cudaGetDeviceProperties(&deviceProp, context.nCudaDeviceId); - if (error != cudaSuccess) - { - throw std::runtime_error("no device properties"); - } - - context.nSharedMemPerBlock = deviceProp.sharedMemPerBlock; - context.nMaxThreadsPerBlock = deviceProp.maxThreadsPerBlock; - context.nMultiProcessorCount = deviceProp.multiProcessorCount; - context.nMaxThreadsPerMultiProcessor = deviceProp.maxThreadsPerMultiProcessor; - - // Refer - https://gitlab-master.nvidia.com/cv/core-modules/tensor_ops/-/merge_requests/48#note_6602087 - context.nReserved0 = 0; - - error = cudaDeviceGetAttribute(&(context.nCudaDevAttrComputeCapabilityMajor), cudaDevAttrComputeCapabilityMajor, - context.nCudaDeviceId); - if (error != cudaSuccess) - { - throw std::runtime_error("no device attribute - nCudaDevAttrComputeCapabilityMajor"); - } - - error = cudaDeviceGetAttribute(&(context.nCudaDevAttrComputeCapabilityMinor), cudaDevAttrComputeCapabilityMinor, - context.nCudaDeviceId); - if (error != cudaSuccess) - { - throw std::runtime_error("no device attribute - nCudaDevAttrComputeCapabilityMinor"); - } -} - -} // anonymous namespace - -struct Context -{ - NppStreamContext nppContext; - size_t time = 0; -}; - -NppStreamContext GetNppStreamContext(cudaStream_t stream) -{ - // Create a memory cache, all timestamp would be initialzed to 0 automatically - static std::array contextCache = {}; - - // Lock the thread - std::lock_guard guard(lock); - - size_t minTimestamp = contextCache[0].time; - size_t minIdx = 0; - for (size_t i = 0; i < CACHE_SIZE; i++) - { - auto &it = contextCache[i]; - if (it.time > 0 && it.nppContext.hStream == stream) - { - it.time = ++timestamp; - return it.nppContext; - } - if (it.time < minTimestamp) - { - minTimestamp = it.time; - minIdx = i; - } - } - auto &it = contextCache[minIdx]; - SetupNppStreamContext(it.nppContext, stream); - it.time = ++timestamp; - return it.nppContext; -} - -}} // namespace cvcore::tensor_ops diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/NppUtils.h b/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/NppUtils.h deleted file mode 100644 index 398ef8f..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/NppUtils.h +++ /dev/null @@ -1,31 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2020-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -#ifndef CVCORE_NPP_UTILS_H -#define CVCORE_NPP_UTILS_H - -#include - -#include - -namespace cvcore { namespace tensor_ops { - -NppStreamContext GetNppStreamContext(cudaStream_t stream); - -}} // namespace cvcore::tensor_ops - -#endif // CVCORE_NPP_UTILS_H diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/OneEuroFilter.cpp b/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/OneEuroFilter.cpp deleted file mode 100644 index 35041b8..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/OneEuroFilter.cpp +++ /dev/null @@ -1,288 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -#include "cv/tensor_ops/OneEuroFilter.h" -#include "cv/core/MathTypes.h" -#include "cv/core/Traits.h" - -#include -#include -#include - -#ifdef NVBENCH_ENABLE -#include -#endif - -namespace cvcore { namespace tensor_ops { - -namespace { - -// 1/(2*PI) -constexpr float kOneOver2Pi = 0.15915494309189533577f; - -// Utilities to get template type from another template type -template -struct deduceDataType; -template -struct deduceDataType::value || std::is_same::value || - std::is_same::value>::type> -{ - typedef float U; -}; -template -struct deduceDataType::value || std::is_same::value || - std::is_same::value>::type> -{ - typedef double U; -}; - -} // namespace - -/* Low pass filter to apply exponential smoothing*/ -template -class LowPassfilter -{ -public: - LowPassfilter() - { - m_firstIteration = true; - } - - void resetState() - { - m_firstIteration = true; - } - - bool isInitialized() const - { - return !m_firstIteration; - } - - T getPreviousValue() const - { - return m_prevfilteredValue; - } - - std::error_code filter(T &outValue, T inValue, float alpha) - { -#ifdef NVBENCH_ENABLE - std::string funcName = "LowPassFilter_"; - std::string tag = funcName + typeid(T).name(); - nv::bench::Timer timerFunc = nv::bench::CPU(tag.c_str(), nv::bench::Flag::DEFAULT); -#endif - if (m_firstIteration) - { - outValue = inValue; - m_firstIteration = false; - } - else - { - outValue = m_prevfilteredValue + (inValue - m_prevfilteredValue) * alpha; - } - m_prevRawValue = inValue; - m_prevfilteredValue = outValue; - return ErrorCode::SUCCESS; - } - -private: - bool m_firstIteration; - T m_prevRawValue; - T m_prevfilteredValue; -}; - -template -struct OneEuroFilterState -{ - // Computes alpha value for the filter - float getAlpha(float dataUpdateRate, float cutOffFreq) const - { - float alpha = cutOffFreq / (dataUpdateRate * kOneOver2Pi + cutOffFreq); - return alpha; - } - - // Resets the parameters and state of the filter - std::error_code resetParams(const OneEuroFilterParams &filterParams) - { - if (filterParams.dataUpdateRate <= 0.0f || filterParams.minCutoffFreq <= 0 || filterParams.derivCutoffFreq <= 0) - { - return ErrorCode::INVALID_ARGUMENT; - } - m_freq = filterParams.dataUpdateRate; - m_mincutoff = filterParams.minCutoffFreq; - m_cutOffSlope = filterParams.cutoffSlope; - m_derivCutOff = filterParams.derivCutoffFreq; - m_alphadxFilt = getAlpha(m_freq, filterParams.derivCutoffFreq); - - xFilt->resetState(); - dxFilt->resetState(); - - m_currfilteredValue = 0.0f; - m_prevfilteredValue = m_currfilteredValue; - return ErrorCode::SUCCESS; - } - - // Constructor for each filter state - OneEuroFilterState(const OneEuroFilterParams &filterParams) - { - xFilt.reset(new LowPassfilter()); - dxFilt.reset(new LowPassfilter()); - auto err = resetParams(filterParams); - if (err != make_error_code(ErrorCode::SUCCESS)) - { - throw err; - } - } - - std::error_code filter(U &outValue, U value) - { -#ifdef NVBENCH_ENABLE - std::string funcName = "OneEuroFilterState_"; - std::string tag = funcName + typeid(U).name(); - nv::bench::Timer timerFunc = nv::bench::CPU(tag.c_str(), nv::bench::Flag::DEFAULT); -#endif - m_prevfilteredValue = m_currfilteredValue; - U dxValue = xFilt->isInitialized() ? (value - xFilt->getPreviousValue()) * m_freq : 0.0f; - U edxValue; - auto err = dxFilt->filter(edxValue, dxValue, m_alphadxFilt); - if (err != make_error_code(ErrorCode::SUCCESS)) - { - return err; - } - // Update the new cutoff frequency - U newCutoff = m_mincutoff + m_cutOffSlope * fabsf(edxValue); - float newAlpha = getAlpha(m_freq, newCutoff); - err = xFilt->filter(m_currfilteredValue, value, newAlpha); - if (err != make_error_code(ErrorCode::SUCCESS)) - { - return err; - } - - outValue = m_currfilteredValue; - return ErrorCode::SUCCESS; - } - std::unique_ptr> xFilt; - std::unique_ptr> dxFilt; - float m_alphadxFilt; - float m_freq; - float m_mincutoff; - float m_cutOffSlope; - float m_derivCutOff; - U m_prevfilteredValue; - U m_currfilteredValue; -}; - -template -struct OneEuroFilter::OneEuroFilterImpl -{ - typedef typename deduceDataType::U DT; - OneEuroFilterImpl(const OneEuroFilterParams &filterParams) - { - size_t numStates = traits::get_dim::value; - m_states.resize(numStates); - for (size_t i = 0; i < m_states.size(); i++) - { - m_states[i].reset(new OneEuroFilterState
(filterParams)); - } - } - - std::error_code resetParams(const OneEuroFilterParams &filterParams) - { - std::error_code err = ErrorCode::SUCCESS; - for (size_t i = 0; i < m_states.size(); i++) - { - err = m_states[i]->resetParams(filterParams); - if (err != make_error_code(ErrorCode::SUCCESS)) - { - return err; - } - } - return ErrorCode::SUCCESS; - } - - ~OneEuroFilterImpl() {} - - template::value == 1>::type * = nullptr> - std::error_code filter(U &outValue, U value) - { - if (m_states.size() != 1) - { - return ErrorCode::INVALID_OPERATION; - } - std::error_code err = m_states[0]->filter(outValue, value); - return err; - } - - template::value != 1>::type * = nullptr> - std::error_code filter(U &outValue, U value) - { - if (m_states.size() <= 1) - { - return ErrorCode::INVALID_OPERATION; - } - std::error_code err = ErrorCode::SUCCESS; - for (size_t i = 0; i < m_states.size(); i++) - { - err = m_states[i]->filter(outValue[i], value[i]); - if (err != make_error_code(ErrorCode::SUCCESS)) - { - return err; - } - } - - return err; - } - - std::vector>> m_states; -}; - -template -OneEuroFilter::OneEuroFilter(const OneEuroFilterParams &filterParams) - : m_pImpl(new OneEuroFilterImpl(filterParams)) -{ -} - -template -OneEuroFilter::~OneEuroFilter() -{ -} - -template -std::error_code OneEuroFilter::resetParams(const OneEuroFilterParams &filterParams) -{ - auto err = m_pImpl->resetParams(filterParams); - return err; -} - -template -std::error_code OneEuroFilter::execute(T &filteredValue, T inValue) -{ -#ifdef NVBENCH_ENABLE - std::string funcName = "OneEuroFilter_"; - std::string tag = funcName + typeid(T).name(); - nv::bench::Timer timerFunc = nv::bench::CPU(tag.c_str(), nv::bench::Flag::DEFAULT); -#endif - auto err = m_pImpl->filter(filteredValue, inValue); - return err; -} - -template class OneEuroFilter; -template class OneEuroFilter; -template class OneEuroFilter; -template class OneEuroFilter; -template class OneEuroFilter; -template class OneEuroFilter; -}} // namespace cvcore::tensor_ops diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/TensorOperators.cpp b/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/TensorOperators.cpp deleted file mode 100644 index 02d34ca..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/TensorOperators.cpp +++ /dev/null @@ -1,116 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -#include "cv/tensor_ops/TensorOperators.h" - -#include - -#ifdef ENABLE_VPI -#include "vpi/VPITensorOperators.h" -#endif - -namespace cvcore { namespace tensor_ops { - -typename TensorContextFactory::MultitonType TensorContextFactory::instances; -std::mutex TensorContextFactory::instanceMutex; - -std::error_code TensorContextFactory::CreateContext(TensorOperatorContext &tensorContext, TensorBackend backend) -{ - using PairType = typename TensorContextFactory::MultitonType::mapped_type; - using CounterType = typename PairType::first_type; - using ValuePtrType = typename PairType::second_type; - - std::lock_guard instanceLock(instanceMutex); - - std::error_code result = ErrorCode::SUCCESS; - - tensorContext = nullptr; - - auto contextItr = instances.find(backend); - if (contextItr == instances.end() && IsBackendSupported(backend)) - { - switch (backend) - { - case TensorBackend::VPI: -#ifdef ENABLE_VPI - try - { - instances[backend] = std::make_pair(1, ValuePtrType(new VPITensorContext{})); - } - catch (std::error_code &e) - { - result = e; - } - catch (...) - { - result = ErrorCode::INVALID_OPERATION; - } -#else // _WIN32 - result = ErrorCode::NOT_IMPLEMENTED; -#endif // _WIN32 - break; - default: - result = ErrorCode::NOT_IMPLEMENTED; - break; - } - tensorContext = instances[backend].second.get(); - } - else - { - contextItr->second.first++; - tensorContext = contextItr->second.second.get(); - } - - return result; -} - -std::error_code TensorContextFactory::DestroyContext(TensorOperatorContext &context) -{ - std::lock_guard instanceLock(instanceMutex); - - auto backend = context->Backend(); - context = nullptr; - auto contextItr = instances.find(backend); - if (contextItr != instances.end()) - { - contextItr->second.first--; - if (contextItr->second.first == 0) - { - instances.erase(backend); - } - } - return ErrorCode::SUCCESS; -} - -bool TensorContextFactory::IsBackendSupported(TensorBackend backend) -{ - bool result = false; - - switch (backend) - { - case TensorBackend::VPI: -#ifdef ENABLE_VPI - result = true; -#endif // _WIN32 - break; - default: - break; - } - - return result; -} -}} // namespace cvcore::tensor_ops diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/vpi/VPIColorConvertImpl.cpp b/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/vpi/VPIColorConvertImpl.cpp deleted file mode 100644 index 693ca5c..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/vpi/VPIColorConvertImpl.cpp +++ /dev/null @@ -1,135 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 -#include -#include -#include -#include - -#include "cv/core/CameraModel.h" -#include "cv/core/Image.h" -#include "cv/core/Memory.h" -#include "cv/core/ProfileUtils.h" -#include "cv/core/Tensor.h" -#include "cv/tensor_ops/ImageUtils.h" - -#include "VPIColorConvertImpl.h" -#include "VPIEnumMapping.h" -#include "VPIStatusMapping.h" - -// VPI includes -#include -#include -#include -#include - -#ifdef NVBENCH_ENABLE -#include -#endif - -namespace cvcore { namespace tensor_ops { - -VPITensorStream::VPIColorConvertImpl::VPIColorConvertImpl() - : m_inputImage(nullptr) - , m_outputImage(nullptr) -{ - std::memset(reinterpret_cast(&m_inputImageData), 0, sizeof(VPIImageData)); - std::memset(reinterpret_cast(&m_outputImageData), 0, sizeof(VPIImageData)); -} - -template -std::error_code VPITensorStream::VPIColorConvertImpl::execute(Image &outputImage, const Image &inputImage, - VPIStream &stream, VPIBackend backend) -{ - std::error_code errCode = make_error_code(VPIStatus::VPI_SUCCESS); - - bool paramsChanged = m_inputImage == nullptr || m_outputImage == nullptr || - CheckParamsChanged(m_inputImageData, inputImage) || - CheckParamsChanged(m_outputImageData, outputImage); - if (paramsChanged) - { - DestroyVPIImageWrapper(m_inputImage, m_inputImageData); - DestroyVPIImageWrapper(m_outputImage, m_outputImageData); - errCode = CreateVPIImageWrapper(m_inputImage, m_inputImageData, inputImage, backend); - if (errCode == make_error_code(VPI_SUCCESS)) - { - errCode = CreateVPIImageWrapper(m_outputImage, m_outputImageData, outputImage, backend); - } - } - - if (errCode == make_error_code(VPIStatus::VPI_SUCCESS)) - { - errCode = UpdateImage(m_inputImage, m_inputImageData, inputImage); - } - if (errCode == make_error_code(VPIStatus::VPI_SUCCESS)) - { - errCode = UpdateImage(m_outputImage, m_outputImageData, outputImage); - } - - if (errCode == make_error_code(VPIStatus::VPI_SUCCESS)) - { -#ifdef NVBENCH_ENABLE - std::string tag = "VPIColorConvert_" + GetMemoryTypeAsString(inputImage.isCPU()) +"Input_" + GetMemoryTypeAsString(outputImage.isCPU()) +"Output_" + getVPIBackendString(backend) + "Backend"; - nv::bench::Timer timerFunc = - nv::bench::VPI(tag.c_str(), nv::bench::Flag::DEFAULT, stream); -#endif - errCode = make_error_code(vpiSubmitConvertImageFormat(stream, backend, m_inputImage, m_outputImage, nullptr)); - } - - if (errCode == make_error_code(VPIStatus::VPI_SUCCESS)) - { - errCode = make_error_code(vpiStreamSync(stream)); - } - - if (errCode != make_error_code(VPIStatus::VPI_SUCCESS)) - { - return errCode; - } - - return make_error_code(ErrorCode::SUCCESS); -} - -VPITensorStream::VPIColorConvertImpl::~VPIColorConvertImpl() -{ - // Destroy Input VPIImage - DestroyVPIImageWrapper(m_inputImage, m_inputImageData); - - // Destroy Output VPIImage - DestroyVPIImageWrapper(m_outputImage, m_outputImageData); -} - -template std::error_code VPITensorStream::VPIColorConvertImpl::execute(Image &, const Image &, - VPIStream &, VPIBackend); -template std::error_code VPITensorStream::VPIColorConvertImpl::execute(Image &, const Image &, - VPIStream &, VPIBackend); -template std::error_code VPITensorStream::VPIColorConvertImpl::execute(Image &, const Image &, - VPIStream &, VPIBackend); -template std::error_code VPITensorStream::VPIColorConvertImpl::execute(Image &, const Image &, - VPIStream &, VPIBackend); -template std::error_code VPITensorStream::VPIColorConvertImpl::execute(Image &, const Image &, - VPIStream &, VPIBackend); -template std::error_code VPITensorStream::VPIColorConvertImpl::execute(Image &, const Image &, - VPIStream &, VPIBackend); -template std::error_code VPITensorStream::VPIColorConvertImpl::execute(Image &, const Image &, - VPIStream &, VPIBackend); -template std::error_code VPITensorStream::VPIColorConvertImpl::execute(Image &, const Image &, - VPIStream &, VPIBackend); -template std::error_code VPITensorStream::VPIColorConvertImpl::execute(Image &, const Image &, - VPIStream &, VPIBackend); -template std::error_code VPITensorStream::VPIColorConvertImpl::execute(Image &, const Image &, - VPIStream &, VPIBackend); - -}} // namespace cvcore::tensor_ops diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/vpi/VPIColorConvertImpl.h b/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/vpi/VPIColorConvertImpl.h deleted file mode 100644 index 0e71266..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/vpi/VPIColorConvertImpl.h +++ /dev/null @@ -1,65 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -#ifndef CVCORE_VPI_COLOR_CONVERT_IMPL_H -#define CVCORE_VPI_COLOR_CONVERT_IMPL_H - -#include "VPITensorOperators.h" -#include "cv/tensor_ops/ITensorOperatorStream.h" -#include "cv/tensor_ops/ImageUtils.h" - -#include - -namespace cvcore { namespace tensor_ops { - -/** - * Color convert implementation for VPI backend. - */ -class VPITensorStream::VPIColorConvertImpl -{ -public: - /** - * Image color conversion constructor. - */ - VPIColorConvertImpl(); - - /** - * Image color conversion a given image type. - * @param outputImage Output image. - * @param inputImage Input image. - * @param stream specified VPI stream. - * @param backend specified VPI backend. - */ - template - std::error_code execute(Image &outputImage, const Image &inputImage, VPIStream &stream, - VPIBackend backend); - - /** - * Image color conversion destroy function to deallocate resources. - */ - ~VPIColorConvertImpl(); - -private: - VPIImage m_inputImage; - VPIImage m_outputImage; - VPIImageData m_inputImageData; - VPIImageData m_outputImageData; -}; - -}} // namespace cvcore::tensor_ops - -#endif //CVCORE_VPI_COLOR_CONVERT_IMPL_H diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/vpi/VPIEnumMapping.h b/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/vpi/VPIEnumMapping.h deleted file mode 100644 index d611167..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/vpi/VPIEnumMapping.h +++ /dev/null @@ -1,196 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -#ifndef CVCORE_VPIENUMMAPPING_H -#define CVCORE_VPIENUMMAPPING_H - -#include - -#include "VPITensorOperators.h" - -namespace cvcore { namespace tensor_ops { - -constexpr VPIBackend ToVpiBackendType(const ComputeEngine &computeEngine) -{ - switch (computeEngine) - { - case ComputeEngine::CPU: - return VPIBackend::VPI_BACKEND_CPU; - case ComputeEngine::PVA: - return VPIBackend::VPI_BACKEND_PVA; - case ComputeEngine::GPU: - return VPIBackend::VPI_BACKEND_CUDA; - case ComputeEngine::VIC: - return VPIBackend::VPI_BACKEND_VIC; - case ComputeEngine::NVENC: - return VPIBackend::VPI_BACKEND_NVENC; - default: - return VPIBackend::VPI_BACKEND_INVALID; - } -} - -constexpr VPIInterpolationType ToVpiInterpolationType(InterpolationType value) -{ - VPIInterpolationType result = VPI_INTERP_NEAREST; - - switch (value) - { - case INTERP_NEAREST: - result = VPI_INTERP_NEAREST; - break; - case INTERP_LINEAR: - result = VPI_INTERP_LINEAR; - break; - case INTERP_CUBIC_CATMULLROM: - result = VPI_INTERP_CATMULL_ROM; - break; - default: - break; - } - - return result; -} - -constexpr VPIBorderExtension ToVpiBorderType(BorderType value) -{ - VPIBorderExtension result = VPI_BORDER_ZERO; - - switch (value) - { - case BORDER_ZERO: - result = VPI_BORDER_ZERO; - break; - case BORDER_REPEAT: - result = VPI_BORDER_CLAMP; - break; - case BORDER_REVERSE: - result = VPI_BORDER_REFLECT; - break; - case BORDER_MIRROR: - result = VPI_BORDER_MIRROR; - break; - default: - break; - } - - return result; -} - -constexpr VPIImageFormat ToVpiImageFormat(ImageType value) -{ - VPIImageFormat result = VPI_IMAGE_FORMAT_Y8_ER; - - switch (value) - { - case Y_U8: - result = VPI_IMAGE_FORMAT_Y8_ER; - break; - case Y_U16: - result = VPI_IMAGE_FORMAT_Y16_ER; - break; - case Y_S8: - result = VPI_IMAGE_FORMAT_S8; - break; - case Y_S16: - result = VPI_IMAGE_FORMAT_S16; - break; - case Y_F32: - result = VPI_IMAGE_FORMAT_F32; - break; - case RGB_U8: - result = VPI_IMAGE_FORMAT_RGB8; - break; - case BGR_U8: - result = VPI_IMAGE_FORMAT_BGR8; - break; - case RGBA_U8: - result = VPI_IMAGE_FORMAT_RGBA8; - break; - case NV12: - result = VPI_IMAGE_FORMAT_NV12_ER; - break; - case NV24: - result = VPI_IMAGE_FORMAT_NV24_ER; - break; - default: - break; - } - - return result; -} - -constexpr VPIPixelType ToVpiPixelType(ImageType value) -{ - VPIPixelType result = VPI_PIXEL_TYPE_U8; - - switch (value) - { - case Y_U8: - result = VPI_PIXEL_TYPE_U8; - break; - case Y_U16: - result = VPI_PIXEL_TYPE_U16; - break; - case Y_S8: - result = VPI_PIXEL_TYPE_S8; - break; - case Y_S16: - result = VPI_PIXEL_TYPE_S16; - break; - case Y_F32: - result = VPI_PIXEL_TYPE_F32; - break; - case RGB_U8: - result = VPI_PIXEL_TYPE_3U8; - break; - case BGR_U8: - result = VPI_PIXEL_TYPE_3U8; - break; - case RGBA_U8: - result = VPI_PIXEL_TYPE_4U8; - break; - default: - break; - } - - return result; -} - -static inline std::string getVPIBackendString(VPIBackend vpiBackend) -{ - switch (vpiBackend) - { - case VPIBackend::VPI_BACKEND_CPU: - return "CPU"; - case VPIBackend::VPI_BACKEND_CUDA: - return "GPU"; - case VPIBackend::VPI_BACKEND_VIC: - return "VIC"; - case VPIBackend::VPI_BACKEND_PVA: - return "PVA"; - case VPIBackend::VPI_BACKEND_NVENC: - return "NVENC"; - case VPIBackend::VPI_BACKEND_INVALID: - return "INVALID"; - default: - return "INVALID"; - } -} - -}} // namespace cvcore::tensor_ops - -#endif // CVCORE_VPIENUMMAPPING_H diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/vpi/VPIImageWarp.h b/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/vpi/VPIImageWarp.h deleted file mode 100644 index 71d098b..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/vpi/VPIImageWarp.h +++ /dev/null @@ -1,37 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 -#ifndef CVCORE_VPIIMAGEWARP_H -#define CVCORE_VPIIMAGEWARP_H -#include -#include - -#include "cv/tensor_ops/IImageWarp.h" -#include "cv/tensor_ops/Errors.h" -#include - -namespace cvcore { namespace tensor_ops { - -struct VPIImageWarp : public IImageWarp -{ - ~VPIImageWarp() = default; - - VPIPayload payload; -}; - -}} // namespace cvcore::tensor_ops - -#endif // CVCORE_VPIIMAGEWARP_H diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/vpi/VPIRemapImpl.cpp b/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/vpi/VPIRemapImpl.cpp deleted file mode 100644 index cb544fd..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/vpi/VPIRemapImpl.cpp +++ /dev/null @@ -1,160 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -#include -#include - -#include "VPIRemapImpl.h" -#include "VPIEnumMapping.h" -#include "VPIStatusMapping.h" - -#include "cv/core/CameraModel.h" -#include "cv/core/Image.h" - -#include -#include - -#ifdef NVBENCH_ENABLE -#include -#endif - -namespace cvcore { namespace tensor_ops { - -VPITensorStream::VPIRemapImpl::VPIRemapImpl() - : m_inputImage(nullptr) - , m_outputImage(nullptr) -{ - std::memset(reinterpret_cast(&m_inputImageData), 0, sizeof(VPIImageData)); - std::memset(reinterpret_cast(&m_outputImageData), 0, sizeof(VPIImageData)); -} - -template -std::error_code VPITensorStream::VPIRemapImpl::initialize(Image & outImage, - const Image & inImage, - VPIBackend backend) -{ - std::error_code status; - status = CreateVPIImageWrapper(m_inputImage, m_inputImageData, inImage, backend); - if(status == make_error_code(VPI_SUCCESS)) - { - status = CreateVPIImageWrapper(m_outputImage, m_outputImageData, outImage, backend); - } - - return status; -} -template std::error_code VPITensorStream::VPIRemapImpl::initialize(Image & outImage, - const Image & inImage, VPIBackend); -template std::error_code VPITensorStream::VPIRemapImpl::initialize(Image & outImage, - const Image & inImage, VPIBackend); -template std::error_code VPITensorStream::VPIRemapImpl::initialize(Image & outImage, - const Image & inImage, VPIBackend); -template std::error_code VPITensorStream::VPIRemapImpl::initialize(Image & outImage, - const Image & inImage, VPIBackend); - -// ----------------------------------------------------------------------------- - -template -std::error_code VPITensorStream::VPIRemapImpl::execute(Image & outImage, - const Image & inImage, - const VPIImageWarp * warp, - InterpolationType interpolation, - BorderType border, - VPIStream & stream, - VPIBackend backend) -{ - std::error_code status = make_error_code(VPI_SUCCESS); - VPIInterpolationType vpiInterpolationType = ToVpiInterpolationType(interpolation); - VPIBorderExtension vpiBorderExt = ToVpiBorderType(border); - - bool paramsChanged = m_inputImage == nullptr || m_outputImage == nullptr || - CheckParamsChanged(m_inputImageData, inImage) || - CheckParamsChanged(m_outputImageData, outImage); - - if(paramsChanged) - { - DestroyVPIImageWrapper(m_inputImage, m_inputImageData); - DestroyVPIImageWrapper(m_outputImage, m_outputImageData); - status = initialize(outImage, inImage, backend); - } - - if(status == make_error_code(VPI_SUCCESS)) - { - status = UpdateImage(m_inputImage, m_inputImageData, inImage); - } - - if(status == make_error_code(VPI_SUCCESS)) - { - status = UpdateImage(m_outputImage, m_outputImageData, outImage); - } - - if(status == make_error_code(VPI_SUCCESS)) - { -#ifdef NVBENCH_ENABLE - std::string tag = "VPISubmitRemap_" + GetMemoryTypeAsString(inImage.isCPU()) +"Input_" + GetMemoryTypeAsString(outImage.isCPU()) +"Output_" + getVPIBackendString(backend) + "Backend"; - nv::bench::Timer timerFunc = - nv::bench::VPI(tag.c_str(), nv::bench::Flag::DEFAULT, stream); -#endif - // Submit remap task for Lens Distortion Correction - status = make_error_code(vpiSubmitRemap(stream, backend, warp->payload, - m_inputImage, m_outputImage, vpiInterpolationType, vpiBorderExt, 0)); - } - - if(status == make_error_code(VPI_SUCCESS)) - { - // Wait for remap to complete - status = make_error_code(vpiStreamSync(stream)); - } - return status; -} -template std::error_code VPITensorStream::VPIRemapImpl::execute(Image & outImage, - const Image & inImage, - const VPIImageWarp * warp, - InterpolationType interpolation, - BorderType border, - VPIStream & stream, - VPIBackend backend); -template std::error_code VPITensorStream::VPIRemapImpl::execute(Image & outImage, - const Image & inImage, - const VPIImageWarp * warp, - InterpolationType interpolation, - BorderType border, - VPIStream & stream, - VPIBackend backend); -template std::error_code VPITensorStream::VPIRemapImpl::execute(Image & outImage, - const Image & inImage, - const VPIImageWarp * warp, - InterpolationType interpolation, - BorderType border, - VPIStream & stream, - VPIBackend backend); -template std::error_code VPITensorStream::VPIRemapImpl::execute(Image & outImage, - const Image & inImage, - const VPIImageWarp * warp, - InterpolationType interpolation, - BorderType border, - VPIStream & stream, - VPIBackend backend); -// ----------------------------------------------------------------------------- - -VPITensorStream::VPIRemapImpl::~VPIRemapImpl() -{ - DestroyVPIImageWrapper(m_inputImage, m_inputImageData); - DestroyVPIImageWrapper(m_outputImage, m_outputImageData); -} -// ----------------------------------------------------------------------------- - -}} // namespace cvcore::tensor_ops diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/vpi/VPIRemapImpl.h b/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/vpi/VPIRemapImpl.h deleted file mode 100644 index e129f04..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/vpi/VPIRemapImpl.h +++ /dev/null @@ -1,82 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -#ifndef CVCORE_VPIREMAPIMPL_H -#define CVCORE_VPIREMAPIMPL_H - -#include -#include - -#include "VPITensorOperators.h" -#include "VPIImageWarp.h" - -namespace cvcore { namespace tensor_ops { -/** - * Remap implementation used for Lens Distortion. - */ -class VPITensorStream::VPIRemapImpl -{ - public: - /* VPIRemapImpl constructor */ - VPIRemapImpl(); - - /** - * Remap Intialization. - * @param outputImage Remap output image of Type - * @param inputImage Remap input image of Type - * @param backend Compute backend - * @return Success if intialization is done successfully, otherwise error is returned - */ - template - std::error_code initialize(Image & outImage, - const Image & inImage, - VPIBackend backend); - - /** - * Remap execution function(non-blocking) - * Application is reqiured to call Sync() before accessing the generated Image. - * @param outImage Remap output image of type NV12 - * @param inImage Remap input image of type NV12 - * @param warp Remap warp pointer - * @param interpolation Interpolation type used for remap - * @param border Border type used for remap - * @param stream VPI stream used for remap - * @param backend VPI backend used for remap - * @return Success if remap is submitted successfully, otherwise error is returned - */ - template - std::error_code execute(Image & outImage, - const Image & inImage, - const VPIImageWarp * warp, - InterpolationType interpolation, - BorderType border, - VPIStream & stream, - VPIBackend backend); - - /* VPIRemapImpl destructor to release resources */ - ~VPIRemapImpl(); - - private: - VPIImage m_inputImage; - VPIImage m_outputImage; - VPIImageData m_inputImageData; - VPIImageData m_outputImageData; -}; - -}} // namespace cvcore::tensor_ops - -#endif //CVCORE_VPIREMAPIMPL_H diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/vpi/VPIResizeImpl.cpp b/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/vpi/VPIResizeImpl.cpp deleted file mode 100644 index 1adffa3..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/vpi/VPIResizeImpl.cpp +++ /dev/null @@ -1,139 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 -#include "VPIResizeImpl.h" - -#include -#include -#include -#include - -// VPI includes -#include -#include -#include -#include -#include - -#include "VPIEnumMapping.h" -#include "VPIStatusMapping.h" -#include "cv/core/CameraModel.h" -#include "cv/core/Image.h" - -#ifdef NVBENCH_ENABLE -#include -#endif - -namespace cvcore { namespace tensor_ops { - -template -std::error_code VPITensorStream::VPIResizeImpl::execute(Image &outputImage, const Image &inputImage, - InterpolationType interpolation, BorderType border, - VPIStream &stream, VPIBackend backend) -{ - std::error_code errCode = ErrorCode::SUCCESS; - VPIStatus status = VPIStatus::VPI_SUCCESS; - VPIInterpolationType interpType; - VPIBorderExtension borderExt; - interpType = ToVpiInterpolationType(interpolation); - borderExt = ToVpiBorderType(border); - - bool paramsChanged = m_inputImage == nullptr || m_outputImage == nullptr || - CheckParamsChanged(m_inputImageData, inputImage) || - CheckParamsChanged(m_outputImageData, outputImage); - if (paramsChanged) - { - DestroyVPIImageWrapper(m_inputImage, m_inputImageData); - DestroyVPIImageWrapper(m_outputImage, m_outputImageData); - errCode = CreateVPIImageWrapper(m_inputImage, m_inputImageData, inputImage, backend); - if (errCode == make_error_code(VPI_SUCCESS)) - { - errCode = CreateVPIImageWrapper(m_outputImage, m_outputImageData, outputImage, backend); - } - } - else - { - - errCode = UpdateImage(m_inputImage, m_inputImageData, inputImage); - if (errCode == make_error_code(VPIStatus::VPI_SUCCESS)) - { - errCode = UpdateImage(m_outputImage, m_outputImageData, outputImage); - } - } - - if (status == VPIStatus::VPI_SUCCESS) - { -#ifdef NVBENCH_ENABLE - std::string tag = "VPISubmitRescale_" + GetMemoryTypeAsString(inputImage.isCPU()) +"Input_" + GetMemoryTypeAsString(outputImage.isCPU()) +"Output_" + getVPIBackendString(backend) + "Backend"; - nv::bench::Timer timerFunc = - nv::bench::VPI(tag.c_str(), nv::bench::Flag::DEFAULT, stream); -#endif - // Resize - status = vpiSubmitRescale(stream, backend, m_inputImage, m_outputImage, interpType, borderExt, 0); - } - - if (status == VPIStatus::VPI_SUCCESS) - { - status = vpiStreamSync(stream); - } - - if (status != VPIStatus::VPI_SUCCESS) - { - return make_error_code(status); - } - return make_error_code(ErrorCode::SUCCESS); -} - -VPITensorStream::VPIResizeImpl::VPIResizeImpl() - : m_inputImage(nullptr) - , m_outputImage(nullptr) -{ - std::memset(reinterpret_cast(&m_inputImageData), 0, sizeof(VPIImageData)); - std::memset(reinterpret_cast(&m_outputImageData), 0, sizeof(VPIImageData)); -} - -/** -* Image resizing destroy function to deallocate resources. -*/ -VPITensorStream::VPIResizeImpl::~VPIResizeImpl() -{ - // Destroy Input VPIImage - DestroyVPIImageWrapper(m_inputImage, m_inputImageData); - // Destroy Output VPIImage - DestroyVPIImageWrapper(m_outputImage, m_outputImageData); -} - -template std::error_code VPITensorStream::VPIResizeImpl::execute(Image &outputImage, - const Image &inputImage, - InterpolationType interpolation, BorderType border, - VPIStream &stream, VPIBackend backend); -template std::error_code VPITensorStream::VPIResizeImpl::execute(Image &outputImage, - const Image &inputImage, - InterpolationType interpolation, BorderType border, - VPIStream &stream, VPIBackend backend); -template std::error_code VPITensorStream::VPIResizeImpl::execute(Image &outputImage, - const Image &inputImage, - InterpolationType interpolation, BorderType border, - VPIStream &stream, VPIBackend backend); -template std::error_code VPITensorStream::VPIResizeImpl::execute(Image &outputImage, - const Image &inputImage, - InterpolationType interpolation, BorderType border, - VPIStream &stream, VPIBackend backend); -template std::error_code VPITensorStream::VPIResizeImpl::execute(Image &outputImage, - const Image &inputImage, - InterpolationType interpolation, BorderType border, - VPIStream &stream, VPIBackend backend); -}} // namespace cvcore::tensor_ops diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/vpi/VPIResizeImpl.h b/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/vpi/VPIResizeImpl.h deleted file mode 100644 index ec20a45..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/vpi/VPIResizeImpl.h +++ /dev/null @@ -1,66 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -#ifndef CVCORE_VPIRESIZEIMPL_H -#define CVCORE_VPIRESIZEIMPL_H - -#include "VPITensorOperators.h" - -#include - -#include "cv/tensor_ops/ITensorOperatorStream.h" -#include "cv/tensor_ops/ImageUtils.h" - -namespace cvcore { namespace tensor_ops { - -/** - * Remap implementation used for Lens Distortion. - */ -class VPITensorStream::VPIResizeImpl -{ -public: - /** - * Initialization for Image resizing. - */ - VPIResizeImpl(); - - /** - * Image resizing a given image type. - * @param outputImage Resize output image of type .RGB_U8 - * @param inputImage Resize input image of type RGB_U8. - * @param type Interpolation type used for resize. - * @param border Image border extension used for resize - */ - template - std::error_code execute(Image &outputImage, const Image &inputImage, InterpolationType interpolation, - BorderType border, VPIStream &stream, VPIBackend backend); - - /** - * Image resizing destroy function to deallocate resources. - */ - ~VPIResizeImpl(); - -private: - VPIImage m_inputImage; - VPIImage m_outputImage; - VPIImageData m_inputImageData; - VPIImageData m_outputImageData; -}; - -}} // namespace cvcore::tensor_ops - -#endif //CVCORE_VPIRESIZEIMPL_H diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/vpi/VPIStatusMapping.cpp b/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/vpi/VPIStatusMapping.cpp deleted file mode 100644 index a43431d..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/vpi/VPIStatusMapping.cpp +++ /dev/null @@ -1,122 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -#include "cv/tensor_ops/Errors.h" -#include "vpi/Status.h" - -#ifndef __cpp_lib_to_underlying -// Using a C++23 feature by hacking std -namespace std -{ - template - constexpr underlying_type_t to_underlying(Enum e) noexcept - { - return static_cast>(e); - } -}; -#endif // __cpp_lib_to_underlying - -namespace cvcore { namespace tensor_ops { - -namespace detail -{ - struct VPIStatusCategory : std::error_category - { - virtual const char * name() const noexcept override final - { - return "vpi-status"; - } - - virtual std::string message(int value) const override final - { - std::string result = "VPI Status"; - - return result; - } - - virtual std::error_condition default_error_condition(int code) const noexcept override final - { - std::error_condition result; - - switch(code) - { - case VPI_SUCCESS: - result = ErrorCode::SUCCESS; - break; - - case VPI_ERROR_INVALID_ARGUMENT: - result = ErrorCode::INVALID_ARGUMENT; - break; - - case VPI_ERROR_INVALID_IMAGE_FORMAT: - result = ErrorCode::INVALID_IMAGE_FORMAT; - break; - - case VPI_ERROR_INVALID_ARRAY_TYPE: - result = ErrorCode::INVALID_STORAGE_TYPE; - break; - - case VPI_ERROR_INVALID_PAYLOAD_TYPE: - result = ErrorCode::INVALID_STORAGE_TYPE; - break; - - case VPI_ERROR_INVALID_OPERATION: - result = ErrorCode::INVALID_OPERATION; - break; - - case VPI_ERROR_INVALID_CONTEXT: - result = ErrorCode::INVALID_ENGINE_TYPE; - break; - - case VPI_ERROR_DEVICE: - result = ErrorCode::DEVICE_ERROR; - break; - - case VPI_ERROR_NOT_READY: - result = ErrorCode::NOT_READY; - break; - - case VPI_ERROR_BUFFER_LOCKED: - result = ErrorCode::SYSTEM_ERROR; - break; - - case VPI_ERROR_OUT_OF_MEMORY: - result = ErrorCode::OUT_OF_MEMORY; - break; - - case VPI_ERROR_INTERNAL: - result = ErrorCode::SYSTEM_ERROR; - break; - - default: - result = ErrorCode::NOT_IMPLEMENTED; - break; - } - - return result; - } - }; -} // namespace detail - -const detail::VPIStatusCategory errorCategory{}; - -std::error_code make_error_code(VPIStatus ec) noexcept -{ - return {std::to_underlying(ec), errorCategory}; -} - -}} // namespace cvcore::tensor_ops diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/vpi/VPIStatusMapping.h b/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/vpi/VPIStatusMapping.h deleted file mode 100644 index 961d4fc..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/vpi/VPIStatusMapping.h +++ /dev/null @@ -1,38 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -#ifndef CVCORE_VPISTATUSMAPPING_H -#define CVCORE_VPISTATUSMAPPING_H - -#include "cv/core/CVError.h" -#include "vpi/Status.h" - -// WARNING: Extending base C++ namespace to cover cvcore error codes -namespace std { - -template <> -struct is_error_code_enum : true_type {}; - -} // namespace std - -namespace cvcore { namespace tensor_ops { - -std::error_code make_error_code(VPIStatus) noexcept; - -}} // namespace cvcore::tensor_ops - -#endif //CVCORE_VPISTATUSMAPPING_H diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/vpi/VPIStereoDisparityEstimatorImpl.cpp b/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/vpi/VPIStereoDisparityEstimatorImpl.cpp deleted file mode 100644 index c4e96e4..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/vpi/VPIStereoDisparityEstimatorImpl.cpp +++ /dev/null @@ -1,211 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -#include -#include - -#include "VPIStereoDisparityEstimatorImpl.h" -#include "VPIEnumMapping.h" -#include "VPIStatusMapping.h" - -#include "cv/core/CameraModel.h" -#include "cv/core/Image.h" - -#include - -#ifdef NVBENCH_ENABLE -#include -#endif - -namespace cvcore { namespace tensor_ops { - -VPITensorStream::VPIStereoDisparityEstimatorImpl::VPIStereoDisparityEstimatorImpl() - : m_inputLeftImage(nullptr) - , m_inputRightImage(nullptr) - , m_outputImage(nullptr) - , m_tempImage(nullptr) - , m_payload(nullptr) - , m_stereoParams() -{ - std::memset(reinterpret_cast(&m_inputLeftImageData), 0, sizeof(VPIImageData)); - std::memset(reinterpret_cast(&m_inputRightImageData), 0, sizeof(VPIImageData)); - std::memset(reinterpret_cast(&m_outputImageData), 0, sizeof(VPIImageData)); - // Disparity values returned from VPI are in Q10.5 format, i.e., signed fixed point with 5 fractional bits. Divide it by 32.0f to convert it to floating point. - vpiInitConvertImageFormatParams(&m_cvtParams); - m_cvtParams.scale = 1.0f / 32; -} - -template -std::error_code VPITensorStream::VPIStereoDisparityEstimatorImpl::initialize(Image &outImage, - const Image &leftImage, - const Image &rightImage, - VPIBackend backend) -{ - std::error_code status; - const std::error_code success = make_error_code(VPI_SUCCESS); - status = CreateVPIImageWrapper(m_inputLeftImage, m_inputLeftImageData, leftImage, backend); - if (status == success) - { - status = CreateVPIImageWrapper(m_inputRightImage, m_inputRightImageData, rightImage, backend); - } - if (status == success) - { - status = CreateVPIImageWrapper(m_outputImage, m_outputImageData, outImage, backend); - } - if (status == success) - { - status = make_error_code( - vpiImageCreate(outImage.getWidth(), outImage.getHeight(), VPI_IMAGE_FORMAT_S16, 0, &m_tempImage)); - } - if (status == success) - { - status = make_error_code(vpiCreateStereoDisparityEstimator(backend, outImage.getWidth(), outImage.getHeight(), - ToVpiImageFormat(T_IN), NULL, &m_payload)); - } - return status; -} - -template std::error_code VPITensorStream::VPIStereoDisparityEstimatorImpl::initialize(Image &outImage, - const Image &leftImage, - const Image &rightImage, - VPIBackend backend); -template std::error_code VPITensorStream::VPIStereoDisparityEstimatorImpl::initialize(Image &outImage, - const Image &leftImage, - const Image &rightImage, - VPIBackend backend); -template std::error_code VPITensorStream::VPIStereoDisparityEstimatorImpl::initialize(Image &outImage, - const Image &leftImage, - const Image &rightImage, - VPIBackend backend); - -// ----------------------------------------------------------------------------- - -template -std::error_code VPITensorStream::VPIStereoDisparityEstimatorImpl::execute(Image &outImage, - const Image &leftImage, - const Image &rightImage, - size_t windowSize, size_t maxDisparity, - VPIStream &stream, VPIBackend backend) -{ - std::error_code status = make_error_code(VPI_SUCCESS); - m_stereoParams.windowSize = static_cast(windowSize); - m_stereoParams.maxDisparity = static_cast(maxDisparity); - - bool paramsChanged = m_inputLeftImage == nullptr || m_inputRightImage == nullptr || m_outputImage == nullptr || - CheckParamsChanged(m_inputLeftImageData, leftImage) || - CheckParamsChanged(m_inputRightImageData, rightImage) || - CheckParamsChanged(m_outputImageData, outImage); - - if (paramsChanged) - { - if (m_payload != nullptr) - { - vpiPayloadDestroy(m_payload); - } - if (m_tempImage != nullptr) - { - vpiImageDestroy(m_tempImage); - } - DestroyVPIImageWrapper(m_inputLeftImage, m_inputLeftImageData); - DestroyVPIImageWrapper(m_inputRightImage, m_inputRightImageData); - DestroyVPIImageWrapper(m_outputImage, m_outputImageData); - - status = initialize(outImage, leftImage, rightImage, backend); - } - - if (status == make_error_code(VPI_SUCCESS)) - { - status = UpdateImage(m_inputLeftImage, m_inputLeftImageData, leftImage); - } - - if (status == make_error_code(VPI_SUCCESS)) - { - status = UpdateImage(m_inputRightImage, m_inputRightImageData, rightImage); - } - - if (status == make_error_code(VPI_SUCCESS)) - { - status = UpdateImage(m_outputImage, m_outputImageData, outImage); - } - - if (status == make_error_code(VPI_SUCCESS)) - { -#ifdef NVBENCH_ENABLE - std::string tag = "VPISubmitStereoDisparityEstimator_" + GetMemoryTypeAsString(leftImage.isCPU()) + "Input_" + - GetMemoryTypeAsString(outImage.isCPU()) + "Output_" + getVPIBackendString(backend) + - "Backend"; - nv::bench::Timer timerFunc = nv::bench::VPI(tag.c_str(), nv::bench::Flag::DEFAULT, stream); -#endif - // Submit SGM task for Stereo Disparity Estimator - status = make_error_code(vpiSubmitStereoDisparityEstimator( - stream, backend, m_payload, m_inputLeftImage, m_inputRightImage, m_tempImage, NULL, &m_stereoParams)); - } - - if (status == make_error_code(VPI_SUCCESS)) - { -#ifdef NVBENCH_ENABLE - std::string tag = "VPISubmitConvertImageFormat_" + GetMemoryTypeAsString(leftImage.isCPU()) + "Input_" + - GetMemoryTypeAsString(outImage.isCPU()) + "Output_" + getVPIBackendString(backend) + - "Backend"; - nv::bench::Timer timerFunc = nv::bench::VPI(tag.c_str(), nv::bench::Flag::DEFAULT, stream); -#endif - // Submit SGM task for Stereo Disparity Estimator - status = - make_error_code(vpiSubmitConvertImageFormat(stream, backend, m_tempImage, m_outputImage, &m_cvtParams)); - } - - if (status == make_error_code(VPI_SUCCESS)) - { - // Wait for stereo disparity estimator to complete - status = make_error_code(vpiStreamSync(stream)); - } - - if (status != make_error_code(VPI_SUCCESS)) - { - return status; - } - return make_error_code(ErrorCode::SUCCESS); -} - -template std::error_code VPITensorStream::VPIStereoDisparityEstimatorImpl::execute( - Image &outImage, const Image &leftImage, const Image &rightImage, size_t windowSize, - size_t maxDisparity, VPIStream &stream, VPIBackend backend); -template std::error_code VPITensorStream::VPIStereoDisparityEstimatorImpl::execute( - Image &outImage, const Image &leftImage, const Image &rightImage, size_t windowSize, - size_t maxDisparity, VPIStream &stream, VPIBackend backend); -template std::error_code VPITensorStream::VPIStereoDisparityEstimatorImpl::execute( - Image &outImage, const Image &leftImage, const Image &rightImage, size_t windowSize, - size_t maxDisparity, VPIStream &stream, VPIBackend backend); -// ----------------------------------------------------------------------------- - -VPITensorStream::VPIStereoDisparityEstimatorImpl::~VPIStereoDisparityEstimatorImpl() -{ - if (m_payload != nullptr) - { - vpiPayloadDestroy(m_payload); - } - if (m_tempImage != nullptr) - { - vpiImageDestroy(m_tempImage); - } - DestroyVPIImageWrapper(m_inputLeftImage, m_inputLeftImageData); - DestroyVPIImageWrapper(m_inputRightImage, m_inputRightImageData); - DestroyVPIImageWrapper(m_outputImage, m_outputImageData); -} -// ----------------------------------------------------------------------------- - -}} // namespace cvcore::tensor_ops diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/vpi/VPIStereoDisparityEstimatorImpl.h b/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/vpi/VPIStereoDisparityEstimatorImpl.h deleted file mode 100644 index 53a5c63..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/vpi/VPIStereoDisparityEstimatorImpl.h +++ /dev/null @@ -1,83 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -#ifndef CVCORE_VPISTEREODISPARITYESTIMATORIMPL_H -#define CVCORE_VPISTEREODISPARITYESTIMATORIMPL_H - -#include -#include -#include - -#include "VPITensorOperators.h" - -namespace cvcore { namespace tensor_ops { -/** - * StereoDisparityEstimator implementation used for stereo dispaity estimate. - */ -class VPITensorStream::VPIStereoDisparityEstimatorImpl -{ -public: - /* VPIStereoDisparityEstimatorImpl constructor */ - VPIStereoDisparityEstimatorImpl(); - - /** - * StereoDisparityEstimator Intialization. - * @param outputImage StereoDisparityEstimator output image - * @param leftImage StereoDisparityEstimator input left image - * @param rightImage StereoDisparityEstimator input right image - * @param backend VPI backend used for StereoDisparityEstimator - * @return Success if intialization is done successfully, otherwise error is returned - */ - template - std::error_code initialize(Image &outImage, const Image &leftImage, const Image &rightImage, - VPIBackend backend); - - /** - * StereoDisparityEstimator execution function(non-blocking) - * Application is reqiured to call Sync() before accessing the generated Image. - * @param outImage StereoDisparityEstimator output image - * @param leftImage StereoDisparityEstimator input left image - * @param rightImage StereoDisparityEstimator input right image - * @param windowSize Represents the median filter size (on PVA+NVENC_VIC backend) or census transform window size (other backends) used in the algorithm - * @param maxDisparity Maximum disparity for matching search - * @param stream VPI stream used for StereoDisparityEstimator - * @param backend VPI backend used for StereoDisparityEstimator - * @return Success if StereoDisparityEstimator is submitted successfully, otherwise error is returned - */ - template - std::error_code execute(Image &outImage, const Image &leftImage, const Image &rightImage, - size_t windowSize, size_t maxDisparity, VPIStream &stream, VPIBackend backend); - - /* VPIStereoDisparityEstimatorImpl destructor to release resources */ - ~VPIStereoDisparityEstimatorImpl(); - -private: - VPIImage m_inputLeftImage; - VPIImage m_inputRightImage; - VPIImage m_outputImage; - VPIImage m_tempImage; - VPIImageData m_inputLeftImageData; - VPIImageData m_inputRightImageData; - VPIImageData m_outputImageData; - VPIPayload m_payload; - VPIStereoDisparityEstimatorParams m_stereoParams; - VPIConvertImageFormatParams m_cvtParams; -}; - -}} // namespace cvcore::tensor_ops - -#endif //CVCORE_VPISTEREODISPARITYESTIMATORIMPL_H diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/vpi/VPITensorOperators.cpp b/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/vpi/VPITensorOperators.cpp deleted file mode 100644 index 48149ef..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/vpi/VPITensorOperators.cpp +++ /dev/null @@ -1,710 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -#include -#include -#include - -#include -#include -#include -#include -#include - -#include "cv/core/CameraModel.h" -#include "cv/core/Image.h" - -#include "VPIColorConvertImpl.h" -#include "VPIEnumMapping.h" -#include "VPIImageWarp.h" -#include "VPIRemapImpl.h" -#include "VPIResizeImpl.h" -#include "VPIStatusMapping.h" -#include "VPIStereoDisparityEstimatorImpl.h" -#include "VPITensorOperators.h" - -#ifdef NVBENCH_ENABLE -#include -#endif - -namespace cvcore { namespace tensor_ops { - -namespace detail { - -// helper function to wrap VPI image for NV12 / NV24 image types -template::value>::type * = nullptr> -std::error_code CreateVPIImageWrapperImpl(VPIImage &vpiImg, VPIImageData &imgdata, const Image &cvcoreImage, VPIBackend backend) -{ -#ifdef NVBENCH_ENABLE - std::string tag = "CreateVPIImageWrapper_NV12/NV24"; - nv::bench::Timer timerFunc = nv::bench::CPU(tag.c_str(), nv::bench::Flag::DEFAULT); -#endif - - std::memset(reinterpret_cast(&imgdata), 0, sizeof(VPIImageData)); - - imgdata.bufferType = cvcoreImage.isCPU() ? VPI_IMAGE_BUFFER_HOST_PITCH_LINEAR : VPI_IMAGE_BUFFER_CUDA_PITCH_LINEAR; - imgdata.buffer.pitch.format = ToVpiImageFormat(T); - imgdata.buffer.pitch.numPlanes = 2; - imgdata.buffer.pitch.planes[0].data = const_cast(cvcoreImage.getLumaData()); - imgdata.buffer.pitch.planes[0].height = cvcoreImage.getLumaHeight(); - imgdata.buffer.pitch.planes[0].width = cvcoreImage.getLumaWidth(); - imgdata.buffer.pitch.planes[0].pixelType = VPI_PIXEL_TYPE_U8; - imgdata.buffer.pitch.planes[0].pitchBytes = cvcoreImage.getLumaStride(TensorDimension::HEIGHT) * sizeof(uint8_t); - imgdata.buffer.pitch.planes[1].data = const_cast(cvcoreImage.getChromaData()); - imgdata.buffer.pitch.planes[1].height = cvcoreImage.getChromaHeight(); - imgdata.buffer.pitch.planes[1].width = cvcoreImage.getChromaWidth(); - imgdata.buffer.pitch.planes[1].pixelType = VPI_PIXEL_TYPE_2U8; - imgdata.buffer.pitch.planes[1].pitchBytes = cvcoreImage.getChromaStride(TensorDimension::HEIGHT) * sizeof(uint8_t); - VPIStatus vpiStatus; - vpiStatus = vpiImageCreateWrapper(&imgdata, nullptr, backend, &vpiImg); - - return make_error_code(vpiStatus); -} - -// helper function to wrap VPI image for interleaved image types -template::value>::type * = nullptr> -std::error_code CreateVPIImageWrapperImpl(VPIImage &vpiImg, VPIImageData &imgdata, const Image &cvcoreImage, VPIBackend backend) -{ -#ifdef NVBENCH_ENABLE - std::string tag = "CreateVPIImageWrapper_Interleaved"; - nv::bench::Timer timerFunc = nv::bench::CPU(tag.c_str(), nv::bench::Flag::DEFAULT); -#endif - - std::memset(reinterpret_cast(&imgdata), 0, sizeof(VPIImageData)); - - using D = typename Image::DataType; - imgdata.bufferType = cvcoreImage.isCPU() ? VPI_IMAGE_BUFFER_HOST_PITCH_LINEAR : VPI_IMAGE_BUFFER_CUDA_PITCH_LINEAR; - imgdata.buffer.pitch.format = ToVpiImageFormat(T); - imgdata.buffer.pitch.numPlanes = 1; - imgdata.buffer.pitch.planes[0].data = const_cast(cvcoreImage.getData()); - imgdata.buffer.pitch.planes[0].height = cvcoreImage.getHeight(); - imgdata.buffer.pitch.planes[0].width = cvcoreImage.getWidth(); - imgdata.buffer.pitch.planes[0].pixelType = ToVpiPixelType(T); - imgdata.buffer.pitch.planes[0].pitchBytes = cvcoreImage.getStride(TensorDimension::HEIGHT) * GetImageElementSize(T); - VPIStatus vpiStatus; - vpiStatus = vpiImageCreateWrapper(&imgdata, nullptr, backend, &vpiImg); - - return make_error_code(vpiStatus); -} - -// helper function to wrap VPI image for planar image types (TODO: not supported in vpi so far) -template::value>::type * = nullptr> -std::error_code CreateVPIImageWrapperImpl(VPIImage &vpiImg, VPIImageData &imgdata, const Image &cvcoreImage, VPIBackend backend) -{ - return make_error_code(VPI_ERROR_INVALID_IMAGE_FORMAT); -} - -} // namespace detail - -std::error_code VPITensorContext::CreateStream(TensorOperatorStream &tensorStream, const ComputeEngine &computeEngine) -{ - tensorStream = nullptr; - - if (!IsComputeEngineCompatible(computeEngine)) - { - return ErrorCode::INVALID_ENGINE_TYPE; - } - - try - { - tensorStream = new VPITensorStream(computeEngine); - } - catch (std::error_code &e) - { - return e; - } - catch (...) - { - return ErrorCode::INVALID_OPERATION; - } - - return ErrorCode::SUCCESS; -} - -VPITensorStream::VPITensorStream(const ComputeEngine &computeEngine) - : m_resizer(new VPIResizeImpl()) - , m_remapper(new VPIRemapImpl()) - , m_colorConverter(new VPIColorConvertImpl()) - , m_stereoDisparityEstimator(new VPIStereoDisparityEstimatorImpl()) -{ - VPIBackend backend = ToVpiBackendType(computeEngine); - VPIStatus status = vpiStreamCreate(backend, &m_stream); - if (status != VPI_SUCCESS) - { - throw make_error_code(status); - } - m_backend = backend; -} - -VPITensorStream::~VPITensorStream() -{ - vpiStreamDestroy(m_stream); -} - -std::error_code VPITensorContext::DestroyStream(TensorOperatorStream &inputStream) -{ - if (inputStream != nullptr) - { - delete inputStream; - inputStream = nullptr; - } - return ErrorCode::SUCCESS; -} - -bool VPITensorContext::IsComputeEngineCompatible(const ComputeEngine &computeEngine) const noexcept -{ - VPIBackend vpibackend = ToVpiBackendType(computeEngine); - if (vpibackend == VPIBackend::VPI_BACKEND_INVALID) - { - return false; - } - return true; -} - -template -std::error_code CreateVPIImageWrapper(VPIImage &vpiImg, VPIImageData &imgdata, const Image &cvcoreImage, VPIBackend backend) -{ - return detail::CreateVPIImageWrapperImpl(vpiImg, imgdata, cvcoreImage, backend); -} - -template std::error_code CreateVPIImageWrapper(VPIImage &, VPIImageData &, const Image &, VPIBackend); -template std::error_code CreateVPIImageWrapper(VPIImage &, VPIImageData &, const Image &, VPIBackend); -template std::error_code CreateVPIImageWrapper(VPIImage &, VPIImageData &, const Image &, VPIBackend); -template std::error_code CreateVPIImageWrapper(VPIImage &, VPIImageData &, const Image &, VPIBackend); -template std::error_code CreateVPIImageWrapper(VPIImage &, VPIImageData &, const Image &, VPIBackend); -template std::error_code CreateVPIImageWrapper(VPIImage &, VPIImageData &, const Image &, VPIBackend); -template std::error_code CreateVPIImageWrapper(VPIImage &, VPIImageData &, const Image &, VPIBackend); -template std::error_code CreateVPIImageWrapper(VPIImage &, VPIImageData &, const Image &, VPIBackend); -template std::error_code CreateVPIImageWrapper(VPIImage &, VPIImageData &, const Image &, VPIBackend); -template std::error_code CreateVPIImageWrapper(VPIImage &, VPIImageData &, const Image &, VPIBackend); - -std::error_code UpdateVPIImageWrapper(VPIImage &image, VPIImageData &imageWrap, bool isCPU) -{ -#ifdef NVBENCH_ENABLE - std::string tag = "UpdateVPIImageWrapper"; - nv::bench::Timer timerFunc = nv::bench::CPU(tag.c_str(), nv::bench::Flag::DEFAULT); -#endif - - VPIStatus status = VPI_SUCCESS; - status = vpiImageSetWrapper(image, &imageWrap); - - return make_error_code(status); -} - -std::error_code DestroyVPIImageWrapper(VPIImage &image, VPIImageData &imageWrap) -{ -#ifdef NVBENCH_ENABLE - std::string tag = "DestroyVPIImageWrapper"; - nv::bench::Timer timerFunc = nv::bench::CPU(tag.c_str(), nv::bench::Flag::DEFAULT); -#endif - - std::memset(reinterpret_cast(&imageWrap), 0, sizeof(VPIImageData)); - if (image != nullptr) - { - vpiImageDestroy(image); - } - - image = nullptr; - - return ErrorCode::SUCCESS; -} -std::error_code VPITensorStream::Status() noexcept -{ - // TODO Needs to be updated for supporting non blocking function calls. - return ErrorCode::SUCCESS; -} - -std::error_code VPITensorStream::Resize(Image &outputImage, const Image &inputImage, - InterpolationType interpolation, BorderType border) -{ - std::unique_lock scopedLock{m_fence}; -#ifdef NVBENCH_ENABLE - std::string tag = "VPIResize_RGB_U8_" + std::to_string(inputImage.getWidth()) + "X" + - std::to_string(inputImage.getHeight()) + "_" + std::to_string(outputImage.getWidth()) + "X" + - std::to_string(outputImage.getHeight()) + "_" + getVPIBackendString(m_backend) + "Backend_" + - GetMemoryTypeAsString(inputImage.isCPU()) + "Input"; - nv::bench::Timer timerFunc = nv::bench::CPU(tag.c_str(), nv::bench::Flag::DEFAULT); -#endif - - std::error_code err_code; - err_code = m_resizer->execute(outputImage, inputImage, interpolation, border, m_stream, m_backend); - return err_code; -} - -std::error_code VPITensorStream::Resize(Image &outputImage, const Image &inputImage, - InterpolationType interpolation, BorderType border) -{ - std::unique_lock scopedLock{m_fence}; -#ifdef NVBENCH_ENABLE - std::string tag = "VPIResize_NV12_" + std::to_string(inputImage.getLumaWidth()) + "X" + - std::to_string(inputImage.getLumaHeight()) + "_" + std::to_string(outputImage.getLumaWidth()) + - "X" + std::to_string(outputImage.getLumaHeight()) + "_" + getVPIBackendString(m_backend) + - "Backend_" + GetMemoryTypeAsString(inputImage.isCPU()) + "Input"; - ; - nv::bench::Timer timerFunc = nv::bench::CPU(tag.c_str(), nv::bench::Flag::DEFAULT); -#endif - - std::error_code err_code; - err_code = m_resizer->execute(outputImage, inputImage, interpolation, border, m_stream, m_backend); - return err_code; -} - -std::error_code VPITensorStream::Resize(Image &outputImage, const Image &inputImage, - InterpolationType interpolation, BorderType border) -{ - std::unique_lock scopedLock{m_fence}; -#ifdef NVBENCH_ENABLE - std::string tag = "VPIResize_RGBA_U8_" + std::to_string(inputImage.getWidth()) + "X" + - std::to_string(inputImage.getHeight()) + "_" + std::to_string(outputImage.getWidth()) + "X" + - std::to_string(outputImage.getHeight()) + "_" + getVPIBackendString(m_backend) + "Backend_" + - GetMemoryTypeAsString(inputImage.isCPU()) + "Input"; - nv::bench::Timer timerFunc = nv::bench::CPU(tag.c_str(), nv::bench::Flag::DEFAULT); -#endif - - std::error_code err_code; - err_code = m_resizer->execute(outputImage, inputImage, interpolation, border, m_stream, m_backend); - return err_code; -} - -std::error_code VPITensorStream::Resize(Image &outputImage, const Image &inputImage, - InterpolationType interpolation, BorderType border) -{ - std::unique_lock scopedLock{m_fence}; -#ifdef NVBENCH_ENABLE - std::string tag = "VPIResize_BGR_U8_" + std::to_string(inputImage.getWidth()) + "X" + - std::to_string(inputImage.getHeight()) + "_" + std::to_string(outputImage.getWidth()) + "X" + - std::to_string(outputImage.getHeight()) + "_" + getVPIBackendString(m_backend) + "Backend_" + - GetMemoryTypeAsString(inputImage.isCPU()) + "Input"; - nv::bench::Timer timerFunc = nv::bench::CPU(tag.c_str(), nv::bench::Flag::DEFAULT); -#endif - - std::error_code err_code; - err_code = m_resizer->execute(outputImage, inputImage, interpolation, border, m_stream, m_backend); - return err_code; -} - -std::error_code VPITensorStream::Resize(Image &outputImage, const Image &inputImage, - InterpolationType interpolation, BorderType border) -{ - std::unique_lock scopedLock{m_fence}; -#ifdef NVBENCH_ENABLE - std::string tag = "VPIResize_NV24_" + std::to_string(inputImage.getLumaWidth()) + "X" + - std::to_string(inputImage.getLumaHeight()) + "_" + std::to_string(outputImage.getLumaWidth()) + - "X" + std::to_string(outputImage.getLumaHeight()) + "_" + getVPIBackendString(m_backend) + - "Backend_" + GetMemoryTypeAsString(inputImage.isCPU()) + "Input"; - nv::bench::Timer timerFunc = nv::bench::CPU(tag.c_str(), nv::bench::Flag::DEFAULT); -#endif - - std::error_code err_code; - err_code = m_resizer->execute(outputImage, inputImage, interpolation, border, m_stream, m_backend); - return err_code; -} - -std::error_code VPITensorStream::Remap(Image &outputImage, const Image &inputImage, - const ImageWarp warp, InterpolationType interpolation, BorderType border) -{ - std::unique_lock scopedLock{m_fence}; -#ifdef NVBENCH_ENABLE - std::string tag = "VPIRemap_RGB_U8_" + std::to_string(inputImage.getWidth()) + "X" + - std::to_string(inputImage.getHeight()) + "_" + std::to_string(outputImage.getWidth()) + "X" + - std::to_string(outputImage.getHeight()) + "_" + getVPIBackendString(m_backend) + "Backend_" + - GetMemoryTypeAsString(inputImage.isCPU()) + "Input"; - nv::bench::Timer timerFunc = nv::bench::CPU(tag.c_str(), nv::bench::Flag::DEFAULT); -#endif - - return m_remapper->execute(outputImage, inputImage, reinterpret_cast(warp), interpolation, border, - m_stream, m_backend); -} - -std::error_code VPITensorStream::Remap(Image &outputImage, const Image &inputImage, - const ImageWarp warp, InterpolationType interpolation, BorderType border) -{ - std::unique_lock scopedLock{m_fence}; -#ifdef NVBENCH_ENABLE - std::string tag = "VPIRemap_BGR_U8_" + std::to_string(inputImage.getWidth()) + "X" + - std::to_string(inputImage.getHeight()) + "_" + std::to_string(outputImage.getWidth()) + "X" + - std::to_string(outputImage.getHeight()) + "_" + getVPIBackendString(m_backend) + "Backend_" + - GetMemoryTypeAsString(inputImage.isCPU()) + "Input"; - nv::bench::Timer timerFunc = nv::bench::CPU(tag.c_str(), nv::bench::Flag::DEFAULT); -#endif - - return m_remapper->execute(outputImage, inputImage, reinterpret_cast(warp), interpolation, border, - m_stream, m_backend); -} - -std::error_code VPITensorStream::Remap(Image &outputImage, const Image &inputImage, const ImageWarp warp, - InterpolationType interpolation, BorderType border) -{ - std::unique_lock scopedLock{m_fence}; -#ifdef NVBENCH_ENABLE - std::string tag = "VPIRemap_NV12_" + std::to_string(inputImage.getLumaWidth()) + "X" + - std::to_string(inputImage.getLumaHeight()) + "_" + std::to_string(outputImage.getLumaWidth()) + - "X" + std::to_string(outputImage.getLumaHeight()) + "_" + getVPIBackendString(m_backend) + - "Backend_" + GetMemoryTypeAsString(inputImage.isCPU()) + "Input"; - nv::bench::Timer timerFunc = nv::bench::CPU(tag.c_str(), nv::bench::Flag::DEFAULT); -#endif - - return m_remapper->execute(outputImage, inputImage, reinterpret_cast(warp), interpolation, border, - m_stream, m_backend); -} - -std::error_code VPITensorStream::Remap(Image &outputImage, const Image &inputImage, const ImageWarp warp, - InterpolationType interpolation, BorderType border) -{ - std::unique_lock scopedLock{m_fence}; -#ifdef NVBENCH_ENABLE - std::string tag = "VPIRemap_NV24_" + std::to_string(inputImage.getLumaWidth()) + "X" + - std::to_string(inputImage.getLumaHeight()) + "_" + std::to_string(outputImage.getLumaWidth()) + - "X" + std::to_string(outputImage.getLumaHeight()) + "_" + getVPIBackendString(m_backend) + - "Backend_" + GetMemoryTypeAsString(inputImage.isCPU()) + "Input"; - nv::bench::Timer timerFunc = nv::bench::CPU(tag.c_str(), nv::bench::Flag::DEFAULT); -#endif - - return m_remapper->execute(outputImage, inputImage, reinterpret_cast(warp), interpolation, border, - m_stream, m_backend); -} - -std::error_code VPITensorStream::ColorConvert(Image &outputImage, const Image &inputImage) -{ - std::unique_lock scopedLock{m_fence}; -#ifdef NVBENCH_ENABLE - std::string tag = "VPIColorConvert_BGR_RGB_" + std::to_string(inputImage.getWidth()) + "X" + - std::to_string(inputImage.getHeight()) + "_" + std::to_string(outputImage.getWidth()) + "X" + - std::to_string(outputImage.getHeight()) + "_" + getVPIBackendString(m_backend) + "Backend_" + - GetMemoryTypeAsString(inputImage.isCPU()) + "Input"; - nv::bench::Timer timerFunc = nv::bench::CPU(tag.c_str(), nv::bench::Flag::DEFAULT); -#endif - - return m_colorConverter->execute(outputImage, inputImage, m_stream, m_backend); -} - -std::error_code VPITensorStream::ColorConvert(Image &outputImage, const Image &inputImage) -{ - std::unique_lock scopedLock{m_fence}; -#ifdef NVBENCH_ENABLE - std::string tag = "VPIColorConvert_RGB_BGR_" + std::to_string(inputImage.getWidth()) + "X" + - std::to_string(inputImage.getHeight()) + "_" + std::to_string(outputImage.getWidth()) + "X" + - std::to_string(outputImage.getHeight()) + "_" + getVPIBackendString(m_backend) + "Backend_" + - GetMemoryTypeAsString(inputImage.isCPU()) + "Input"; - nv::bench::Timer timerFunc = nv::bench::CPU(tag.c_str(), nv::bench::Flag::DEFAULT); -#endif - - return m_colorConverter->execute(outputImage, inputImage, m_stream, m_backend); -} - -std::error_code VPITensorStream::ColorConvert(Image &outputImage, const Image &inputImage) -{ - std::unique_lock scopedLock{m_fence}; -#ifdef NVBENCH_ENABLE - std::string tag = "VPIColorConvert_NV12_BGR_" + std::to_string(inputImage.getWidth()) + "X" + - std::to_string(inputImage.getHeight()) + "_" + std::to_string(outputImage.getLumaWidth()) + "X" + - std::to_string(outputImage.getLumaHeight()) + "_" + getVPIBackendString(m_backend) + "Backend_" + - GetMemoryTypeAsString(inputImage.isCPU()) + "Input"; - nv::bench::Timer timerFunc = nv::bench::CPU(tag.c_str(), nv::bench::Flag::DEFAULT); -#endif - - return m_colorConverter->execute(outputImage, inputImage, m_stream, m_backend); -} - -std::error_code VPITensorStream::ColorConvert(Image &outputImage, const Image &inputImage) -{ - std::unique_lock scopedLock{m_fence}; -#ifdef NVBENCH_ENABLE - std::string tag = "VPIColorConvert_NV24_BGR_" + std::to_string(inputImage.getWidth()) + "X" + - std::to_string(inputImage.getHeight()) + "_" + std::to_string(outputImage.getLumaWidth()) + "X" + - std::to_string(outputImage.getLumaHeight()) + "_" + getVPIBackendString(m_backend) + "Backend_" + - GetMemoryTypeAsString(inputImage.isCPU()) + "Input"; - nv::bench::Timer timerFunc = nv::bench::CPU(tag.c_str(), nv::bench::Flag::DEFAULT); -#endif - - return m_colorConverter->execute(outputImage, inputImage, m_stream, m_backend); -} - -std::error_code VPITensorStream::ColorConvert(Image &outputImage, const Image &inputImage) -{ - std::unique_lock scopedLock{m_fence}; -#ifdef NVBENCH_ENABLE - std::string tag = "VPIColorConvert_BGR_NV12_" + std::to_string(inputImage.getLumaWidth()) + "X" + - std::to_string(inputImage.getLumaHeight()) + "_" + std::to_string(outputImage.getWidth()) + "X" + - std::to_string(outputImage.getHeight()) + "_" + getVPIBackendString(m_backend) + "Backend_" + - GetMemoryTypeAsString(inputImage.isCPU()) + "Input"; - nv::bench::Timer timerFunc = nv::bench::CPU(tag.c_str(), nv::bench::Flag::DEFAULT); -#endif - - return m_colorConverter->execute(outputImage, inputImage, m_stream, m_backend); -} - -std::error_code VPITensorStream::ColorConvert(Image &outputImage, const Image &inputImage) -{ - std::unique_lock scopedLock{m_fence}; -#ifdef NVBENCH_ENABLE - std::string tag = "VPIColorConvert_BGR_NV24_" + std::to_string(inputImage.getLumaWidth()) + "X" + - std::to_string(inputImage.getLumaHeight()) + "_" + std::to_string(outputImage.getWidth()) + "X" + - std::to_string(outputImage.getHeight()) + "_" + getVPIBackendString(m_backend) + "Backend_" + - GetMemoryTypeAsString(inputImage.isCPU()) + "Input"; - nv::bench::Timer timerFunc = nv::bench::CPU(tag.c_str(), nv::bench::Flag::DEFAULT); -#endif - - return m_colorConverter->execute(outputImage, inputImage, m_stream, m_backend); -} - -std::error_code VPITensorStream::ColorConvert(Image &outputImage, const Image &inputImage) -{ - std::unique_lock scopedLock{m_fence}; -#ifdef NVBENCH_ENABLE - std::string tag = "VPIColorConvert_NV12_RGB_" + std::to_string(inputImage.getWidth()) + "X" + - std::to_string(inputImage.getHeight()) + "_" + std::to_string(outputImage.getLumaWidth()) + "X" + - std::to_string(outputImage.getLumaHeight()) + "_" + getVPIBackendString(m_backend) + "Backend_" + - GetMemoryTypeAsString(inputImage.isCPU()) + "Input"; - nv::bench::Timer timerFunc = nv::bench::CPU(tag.c_str(), nv::bench::Flag::DEFAULT); -#endif - - return m_colorConverter->execute(outputImage, inputImage, m_stream, m_backend); -} - -std::error_code VPITensorStream::ColorConvert(Image &outputImage, const Image &inputImage) -{ - std::unique_lock scopedLock{m_fence}; -#ifdef NVBENCH_ENABLE - std::string tag = "VPIColorConvert_NV24_RGB_" + std::to_string(inputImage.getWidth()) + "X" + - std::to_string(inputImage.getHeight()) + "_" + std::to_string(outputImage.getLumaWidth()) + "X" + - std::to_string(outputImage.getLumaHeight()) + "_" + getVPIBackendString(m_backend) + "Backend_" + - GetMemoryTypeAsString(inputImage.isCPU()) + "Input"; - nv::bench::Timer timerFunc = nv::bench::CPU(tag.c_str(), nv::bench::Flag::DEFAULT); -#endif - - return m_colorConverter->execute(outputImage, inputImage, m_stream, m_backend); -} - -std::error_code VPITensorStream::ColorConvert(Image &outputImage, const Image &inputImage) -{ - std::unique_lock scopedLock{m_fence}; -#ifdef NVBENCH_ENABLE - std::string tag = "VPIColorConvert_RGB_NV12_" + std::to_string(inputImage.getLumaWidth()) + "X" + - std::to_string(inputImage.getLumaHeight()) + "_" + std::to_string(outputImage.getWidth()) + "X" + - std::to_string(outputImage.getHeight()) + "_" + getVPIBackendString(m_backend) + "Backend_" + - GetMemoryTypeAsString(inputImage.isCPU()) + "Input"; - nv::bench::Timer timerFunc = nv::bench::CPU(tag.c_str(), nv::bench::Flag::DEFAULT); -#endif - - return m_colorConverter->execute(outputImage, inputImage, m_stream, m_backend); -} - -std::error_code VPITensorStream::ColorConvert(Image &outputImage, const Image &inputImage) -{ - std::unique_lock scopedLock{m_fence}; -#ifdef NVBENCH_ENABLE - std::string tag = "VPIColorConvert_RGB_NV24_" + std::to_string(inputImage.getLumaWidth()) + "X" + - std::to_string(inputImage.getLumaHeight()) + "_" + std::to_string(outputImage.getWidth()) + "X" + - std::to_string(outputImage.getHeight()) + "_" + getVPIBackendString(m_backend) + "Backend_" + - GetMemoryTypeAsString(inputImage.isCPU()) + "Input"; - nv::bench::Timer timerFunc = nv::bench::CPU(tag.c_str(), nv::bench::Flag::DEFAULT); -#endif - - return m_colorConverter->execute(outputImage, inputImage, m_stream, m_backend); -} - -std::error_code VPITensorStream::StereoDisparityEstimator(Image &outputImage, const Image &inputLeftImage, - const Image &inputRightImage, size_t windowSize, - size_t maxDisparity) -{ - std::unique_lock scopedLock{m_fence}; -#ifdef NVBENCH_ENABLE - std::string tag = "StereoDisparityEstimator_Y_F32_Y_U8_" + std::to_string(inputLeftImage.getWidth()) + "X" + - std::to_string(inputLeftImage.getHeight()) + "_" + std::to_string(outputImage.getWidth()) + "X" + - std::to_string(outputImage.getHeight()) + "_" + getVPIBackendString(m_backend) + "Backend_" + - GetMemoryTypeAsString(inputLeftImage.isCPU()) + "Input"; - nv::bench::Timer timerFunc = nv::bench::CPU(tag.c_str(), nv::bench::Flag::DEFAULT); -#endif - return m_stereoDisparityEstimator->execute(outputImage, inputLeftImage, inputRightImage, windowSize, maxDisparity, - m_stream, m_backend); -} - -std::error_code VPITensorStream::StereoDisparityEstimator(Image &outputImage, const Image &inputLeftImage, - const Image &inputRightImage, size_t windowSize, - size_t maxDisparity) -{ - std::unique_lock scopedLock{m_fence}; -#ifdef NVBENCH_ENABLE - std::string tag = "StereoDisparityEstimator_Y_F32_NV12_" + std::to_string(inputLeftImage.getLumaWidth()) + "X" + - std::to_string(inputLeftImage.getLumaHeight()) + "_" + std::to_string(outputImage.getWidth()) + - "X" + std::to_string(outputImage.getHeight()) + "_" + getVPIBackendString(m_backend) + - "Backend_" + GetMemoryTypeAsString(inputLeftImage.isCPU()) + "Input"; - nv::bench::Timer timerFunc = nv::bench::CPU(tag.c_str(), nv::bench::Flag::DEFAULT); -#endif - return m_stereoDisparityEstimator->execute(outputImage, inputLeftImage, inputRightImage, windowSize, maxDisparity, - m_stream, m_backend); -} - -std::error_code VPITensorStream::StereoDisparityEstimator(Image &outputImage, const Image &inputLeftImage, - const Image &inputRightImage, size_t windowSize, - size_t maxDisparity) -{ - std::unique_lock scopedLock{m_fence}; -#ifdef NVBENCH_ENABLE - std::string tag = "StereoDisparityEstimator_Y_F32_NV24_" + std::to_string(inputLeftImage.getLumaWidth()) + "X" + - std::to_string(inputLeftImage.getLumaHeight()) + "_" + std::to_string(outputImage.getWidth()) + - "X" + std::to_string(outputImage.getHeight()) + "_" + getVPIBackendString(m_backend) + - "Backend_" + GetMemoryTypeAsString(inputLeftImage.isCPU()) + "Input"; - nv::bench::Timer timerFunc = nv::bench::CPU(tag.c_str(), nv::bench::Flag::DEFAULT); -#endif - return m_stereoDisparityEstimator->execute(outputImage, inputLeftImage, inputRightImage, windowSize, maxDisparity, - m_stream, m_backend); -} - -TensorBackend VPITensorContext::Backend() const noexcept -{ - return TensorBackend::VPI; -} - -std::error_code VPITensorStream::GenerateWarpFromCameraModel(ImageWarp &warp, const ImageGrid &grid, - const CameraModel &source, const CameraIntrinsics &target) -{ - std::unique_lock scopedLock{m_fence}; -#ifdef NVBENCH_ENABLE - std::string tag = "GenerateWarpFromCameraModel"; - nv::bench::Timer timerFunc = nv::bench::CPU(tag.c_str(), nv::bench::Flag::DEFAULT); -#endif - - VPIStatus status = VPI_SUCCESS; - - VPIWarpMap map = {0}; - map.grid.numHorizRegions = grid.numHorizRegions; - for (std::size_t i = 0; i < static_cast(grid.numHorizRegions); i++) - { - map.grid.regionWidth[i] = grid.regionWidth[i]; - map.grid.horizInterval[i] = grid.horizInterval[i]; - } - map.grid.numVertRegions = grid.numVertRegions; - for (std::size_t i = 0; i < static_cast(grid.numVertRegions); i++) - { - map.grid.regionHeight[i] = grid.regionHeight[i]; - map.grid.vertInterval[i] = grid.vertInterval[i]; - } - status = vpiWarpMapAllocData(&map); - - if ((status == VPI_SUCCESS) && (map.keypoints)) - { - switch (source.distortion.type) - { - case CameraDistortionType::Polynomial: - { - VPIPolynomialLensDistortionModel distortion; - distortion.k1 = source.distortion.k1; - distortion.k2 = source.distortion.k2; - distortion.k3 = source.distortion.k3; - distortion.k4 = source.distortion.k4; - distortion.k5 = source.distortion.k5; - distortion.k6 = source.distortion.k6; - distortion.p1 = source.distortion.p1; - distortion.p2 = source.distortion.p2; - status = vpiWarpMapGenerateFromPolynomialLensDistortionModel( - source.intrinsic.m_intrinsics, source.extrinsic.m_extrinsics, target.m_intrinsics, &distortion, &map); - break; - } - case CameraDistortionType::FisheyeEquidistant: - { - VPIFisheyeLensDistortionModel distortion; - distortion.k1 = source.distortion.k1; - distortion.k2 = source.distortion.k2; - distortion.k3 = source.distortion.k3; - distortion.k4 = source.distortion.k4; - distortion.mapping = VPI_FISHEYE_EQUIDISTANT; - status = vpiWarpMapGenerateFromFisheyeLensDistortionModel( - source.intrinsic.m_intrinsics, source.extrinsic.m_extrinsics, target.m_intrinsics, &distortion, &map); - break; - } - case CameraDistortionType::FisheyeEquisolid: - { - VPIFisheyeLensDistortionModel distortion; - distortion.k1 = source.distortion.k1; - distortion.k2 = source.distortion.k2; - distortion.k3 = source.distortion.k3; - distortion.k4 = source.distortion.k4; - distortion.mapping = VPI_FISHEYE_EQUISOLID; - status = vpiWarpMapGenerateFromFisheyeLensDistortionModel( - source.intrinsic.m_intrinsics, source.extrinsic.m_extrinsics, target.m_intrinsics, &distortion, &map); - break; - } - case CameraDistortionType::FisheyeOrthoGraphic: - { - VPIFisheyeLensDistortionModel distortion; - distortion.k1 = source.distortion.k1; - distortion.k2 = source.distortion.k2; - distortion.k3 = source.distortion.k3; - distortion.k4 = source.distortion.k4; - distortion.mapping = VPI_FISHEYE_ORTHOGRAPHIC; - status = vpiWarpMapGenerateFromFisheyeLensDistortionModel( - source.intrinsic.m_intrinsics, source.extrinsic.m_extrinsics, target.m_intrinsics, &distortion, &map); - break; - } - case CameraDistortionType::FisheyeStereographic: - { - VPIFisheyeLensDistortionModel distortion; - distortion.k1 = source.distortion.k1; - distortion.k2 = source.distortion.k2; - distortion.k3 = source.distortion.k3; - distortion.k4 = source.distortion.k4; - distortion.mapping = VPI_FISHEYE_STEREOGRAPHIC; - status = vpiWarpMapGenerateFromFisheyeLensDistortionModel( - source.intrinsic.m_intrinsics, source.extrinsic.m_extrinsics, target.m_intrinsics, &distortion, &map); - break; - } - default: - status = VPI_ERROR_INVALID_ARGUMENT; - break; - } - } - - if ((status == VPI_SUCCESS) && (map.keypoints)) - { - if (warp != nullptr) - { - vpiPayloadDestroy(reinterpret_cast(warp)->payload); - delete warp; - } - warp = new VPIImageWarp; - status = vpiCreateRemap(m_backend, &map, &(reinterpret_cast(warp)->payload)); - } - - // Delete map after payload is generated - vpiWarpMapFreeData(&map); - - return make_error_code(status); -} - -std::error_code VPITensorStream::DestroyWarp(ImageWarp &warp) noexcept -{ - std::unique_lock scopedLock{m_fence}; - if (warp != nullptr) - { - try - { - vpiPayloadDestroy(reinterpret_cast(warp)->payload); - } - catch (std::error_code &e) - { - return e; - } - - delete reinterpret_cast(warp); - warp = nullptr; - } - return make_error_code(ErrorCode::SUCCESS); -} - -}} // namespace cvcore::tensor_ops diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/vpi/VPITensorOperators.h b/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/vpi/VPITensorOperators.h deleted file mode 100644 index e01ed11..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/vpi/VPITensorOperators.h +++ /dev/null @@ -1,272 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -#ifndef CVCORE_VPITENSOROPERATORS_H -#define CVCORE_VPITENSOROPERATORS_H - -#include -#include - -#include -#include -#include -#include - -#include "cv/core/CameraModel.h" -#include "cv/tensor_ops/ITensorOperatorContext.h" -#include "cv/tensor_ops/ITensorOperatorStream.h" - -#include "VPIEnumMapping.h" - -namespace cvcore { namespace tensor_ops { - -/** - * Returns the corresponding VPI backend given the cvcore compute engine. - * @param computeEngine Compute engine used. - * @return Returns the VPIbackend. - */ -VPIBackend getVPIBackend(const ComputeEngine &computeEngine); - -/** - * Wraps a CVCore Image type into a VPIImage - * @param vpiImage VPIImage - * @param imgdata VPIImage data - * @param cvcoreImage CVCore image - * @param backend Compute backend - * @return error code - */ -template -std::error_code CreateVPIImageWrapper(VPIImage &vpiImg, VPIImageData &imgdata, const Image &cvcoreImage, VPIBackend backend); - -/** - * Update VPI Image data pointer - * @param vpiImage VPIImage - * @param imgdata VPIImage data - * @param isCPU data is on CPU or GPU - * @return error code - */ -std::error_code UpdateVPIImageWrapper(VPIImage &image, VPIImageData &imageWrap, bool isCPU); - -/** - * Destory Wrapped VPI Image - * @param vpiImage VPIImage - * @param imgdata VPIImage data - * @return error code - */ -std::error_code DestroyVPIImageWrapper(VPIImage &image, VPIImageData &imageWrap); - -/** - * Update VPI Image given CVCORE Image - * @param vpiImage VPIImage - * @param vpiImageData VPIImage data - * @param image CVCORE Image - * @return error code - */ -template::value>::type * = nullptr> -std::error_code UpdateImage(VPIImage &vpiImage, VPIImageData &vpiImageData, const Image &image) -{ - using D = typename Image::DataType; - vpiImageData.buffer.pitch.planes[0].data = const_cast(image.getData()); - return UpdateVPIImageWrapper(vpiImage, vpiImageData, image.isCPU()); -} - -/** - * Update VPI Image given CVCORE Image - * @param vpiImage VPIImage - * @param vpiImageData VPIImage data - * @param image CVCORE Image - * @return error code - */ -template::value>::type * = nullptr> -std::error_code UpdateImage(VPIImage &vpiImage, VPIImageData &vpiImageData, const Image &image) -{ - vpiImageData.buffer.pitch.planes[0].data = const_cast(image.getLumaData()); - vpiImageData.buffer.pitch.planes[1].data = const_cast(image.getChromaData()); - return UpdateVPIImageWrapper(vpiImage, vpiImageData, image.isCPU()); -} - -/** - * Check if params of VPIImageData is consistent with the given CVCORE Image - * @param vpiImageData VPIImage data - * @param image CVCORE Image - * @return whether param changed - */ -template::value && !IsPlanarImage::value>::type * = nullptr> -bool CheckParamsChanged(VPIImageData &vpiImageData, const Image &image) -{ - bool paramsChanged = false; - // Did format change - paramsChanged = paramsChanged || vpiImageData.buffer.pitch.format != ToVpiImageFormat(T); - // Did image dimension change - paramsChanged = - paramsChanged || (vpiImageData.buffer.pitch.planes[0].height != static_cast(image.getHeight()) || - vpiImageData.buffer.pitch.planes[0].width != static_cast(image.getWidth())); - return paramsChanged; -} - -/** - * Check if params of VPIImageData is consistent with the given CVCORE Image - * @param vpiImageData VPIImage data - * @param image CVCORE Image - * @return whether param changed - */ -template::value>::type * = nullptr> -bool CheckParamsChanged(VPIImageData &vpiImageData, const Image &image) -{ - bool paramsChanged = false; - - // Did format change - paramsChanged = paramsChanged || vpiImageData.buffer.pitch.format != ToVpiImageFormat(T); - - // Did image dimension change - paramsChanged = paramsChanged || - (vpiImageData.buffer.pitch.planes[0].height != static_cast(image.getLumaHeight()) || - vpiImageData.buffer.pitch.planes[0].width != static_cast(image.getLumaWidth()) || - vpiImageData.buffer.pitch.planes[1].height != static_cast(image.getChromaHeight()) || - vpiImageData.buffer.pitch.planes[1].width != static_cast(image.getChromaWidth())); - return paramsChanged; -} - -/** - * Implementation of VPITensorContext - */ -class VPITensorContext : public ITensorOperatorContext -{ -public: - /** - * Default Constructor for VPI Context. - */ - VPITensorContext() = default; - - /** - * Default Destructor for VPI Context. - */ - ~VPITensorContext() = default; - - /** - * Creates a stream based on compute engine - * @param computeEngine CVCore Compute engine - * @return Pointer to ITensorOperatorStream object. - */ - virtual std::error_code CreateStream(TensorOperatorStream &, const ComputeEngine &computeEngine) override; - - /** - * Destroy stream creates. - * @param inputStream Input stream to be deleted - * @return Error code - */ - virtual std::error_code DestroyStream(TensorOperatorStream &inputStream) override; - - /** - * Checks if stream type is supported for a given backend. - * @param computeEngine CVCore Compute engine - * @return true if stream type is available. - */ - virtual bool IsComputeEngineCompatible(const ComputeEngine &computeEngine) const noexcept override; - - /** - * Returns the backend type - */ - virtual TensorBackend Backend() const noexcept override; - -private: -}; - -/** - * Implementation of VPITensorStream - */ -class VPITensorStream : public ITensorOperatorStream -{ -public: - virtual std::error_code Status() noexcept override; - - virtual std::error_code GenerateWarpFromCameraModel(ImageWarp &warp, const ImageGrid &grid, - const CameraModel &source, - const CameraIntrinsics &target) override; - virtual std::error_code DestroyWarp(ImageWarp &warp) noexcept override; - - virtual std::error_code Remap(Image &outputImage, const Image &inputImage, const ImageWarp warp, - InterpolationType interpolation, BorderType border) override; - - virtual std::error_code Remap(Image &outputImage, const Image &inputImage, const ImageWarp warp, - InterpolationType interpolation, BorderType border) override; - - virtual std::error_code Remap(Image &outputImage, const Image &inputImage, const ImageWarp warp, - InterpolationType interpolation, BorderType border) override; - - virtual std::error_code Remap(Image &outputImage, const Image &inputImage, const ImageWarp warp, - InterpolationType interpolation, BorderType border) override; - - virtual std::error_code Resize(Image &outputImage, const Image &inputImage, - InterpolationType interpolation, BorderType border) override; - virtual std::error_code Resize(Image &outputImage, const Image &inputImage, - InterpolationType interpolation, BorderType border) override; - virtual std::error_code Resize(Image &outputImage, const Image &inputImage, - InterpolationType interpolation, BorderType border) override; - virtual std::error_code Resize(Image &outputImage, const Image &inputImage, - InterpolationType interpolation, BorderType border) override; - virtual std::error_code Resize(Image &outputImage, const Image &inputImage, - InterpolationType interpolation, BorderType border) override; - - virtual std::error_code ColorConvert(Image &outputImage, const Image &inputImage) override; - virtual std::error_code ColorConvert(Image &outputImage, const Image &inputImage) override; - virtual std::error_code ColorConvert(Image &outputImage, const Image &inputImage) override; - virtual std::error_code ColorConvert(Image &outputImage, const Image &inputImage) override; - virtual std::error_code ColorConvert(Image &outputImage, const Image &inputImage) override; - virtual std::error_code ColorConvert(Image &outputImage, const Image &inputImage) override; - virtual std::error_code ColorConvert(Image &outputImage, const Image &inputImage) override; - virtual std::error_code ColorConvert(Image &outputImage, const Image &inputImage) override; - virtual std::error_code ColorConvert(Image &outputImage, const Image &inputImage) override; - virtual std::error_code ColorConvert(Image &outputImage, const Image &inputImage) override; - virtual std::error_code StereoDisparityEstimator(Image &outputImage, const Image &inputLeftImage, - const Image &inputRightImage, size_t windowSize, - size_t maxDisparity) override; - virtual std::error_code StereoDisparityEstimator(Image &outputImage, const Image &inputLeftImage, - const Image &inputRightImage, size_t windowSize, - size_t maxDisparity) override; - virtual std::error_code StereoDisparityEstimator(Image &outputImage, const Image &inputLeftImage, - const Image &inputRightImage, size_t windowSize, - size_t maxDisparity) override; - -protected: - friend class VPITensorContext; - - VPITensorStream(const ComputeEngine &computeEngine); - ~VPITensorStream(); - -private: - class VPIResizeImpl; - class VPIRemapImpl; - class VPIColorConvertImpl; - class VPIStereoDisparityEstimatorImpl; - - mutable std::mutex m_fence; - - std::unique_ptr m_resizer; - std::unique_ptr m_remapper; - std::unique_ptr m_colorConverter; - std::unique_ptr m_stereoDisparityEstimator; - - VPIStream m_stream; - VPIBackend m_backend; -}; - -}} // namespace cvcore::tensor_ops - -#endif // CVCORE_VPITENSOROPERATORS_H diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/trtbackend/TRTBackend.cpp b/isaac_ros_bi3d/gxf/bi3d/cvcore/src/trtbackend/TRTBackend.cpp deleted file mode 100644 index aa315e5..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/trtbackend/TRTBackend.cpp +++ /dev/null @@ -1,634 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2020-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -#include "cv/trtbackend/TRTBackend.h" - -#include "NvInfer.h" -#include "NvOnnxConfig.h" -#include "NvOnnxParser.h" -#include "NvUffParser.h" -#include "NvUtils.h" - -#include -#include -#include -#include - -namespace cvcore { - -namespace { - - -void WriteSerializedEngineToFile(const char *data, size_t engineSize, std::string &outputFile) -{ - std::ofstream outFile(outputFile.c_str(), std::ios::binary); - if (!outFile.is_open()) - { - throw std::runtime_error("Cannot open file to write serialized Engine. Permissions? "); - } - else - { - outFile.write(data, engineSize); - outFile.close(); - } -} - -nvinfer1::Dims4 GetTRTBlobDimension(int batch, int channels, int height, int width, TRTBackendBlobLayout layout) -{ - nvinfer1::Dims4 dims; - switch (layout) - { - case TRTBackendBlobLayout::PLANAR: - { - dims = {batch, channels, height, width}; - break; - } - case TRTBackendBlobLayout::INTERLEAVED: - { - dims = {batch, height, width, channels}; - break; - } - default: - { - throw std::runtime_error("Only PLANAR and INTERLEAVED types allowed"); - } - } - return dims; -} - -nvinfer1::Dims3 GetTRTBlobDimension(int channels, int height, int width, TRTBackendBlobLayout layout) -{ - nvinfer1::Dims3 dims; - switch (layout) - { - case TRTBackendBlobLayout::PLANAR: - { - dims = {channels, height, width}; - break; - } - case TRTBackendBlobLayout::INTERLEAVED: - { - dims = {height, width, channels}; - break; - } - default: - { - throw std::runtime_error("Only PLANAR and INTERLEAVED types allowed"); - } - } - return dims; -} - -bool SetupProfile(nvinfer1::IOptimizationProfile *profile, nvinfer1::INetworkDefinition *network, - TRTBackendParams ¶ms) -{ - // This shouldn't be hard-coded rather should be set by the user. - int kMINBatchSize = 1; - int kMAXBatchSize = 32; - - if (kMAXBatchSize < params.batchSize) - { - throw std::runtime_error("Max batch size is hard-coded to 32."); - } - - bool hasDynamicShape = false; - for (int i = 0; i < network->getNbInputs(); i++) - { - auto input = network->getInput(i); - nvinfer1::Dims dims = input->getDimensions(); - const bool isDynamicInput = std::any_of(dims.d, dims.d + dims.nbDims, [](int dim) { return dim == -1; }); - if (isDynamicInput) - { - hasDynamicShape = true; - auto it = std::find(params.inputLayers.begin(), params.inputLayers.end(), std::string(input->getName())); - if (it == params.inputLayers.end()) - { - throw std::runtime_error("Undefined dynamic input shape"); - } - int pos = it - params.inputLayers.begin(); - auto inputDim = params.inputDims[pos]; - profile->setDimensions(input->getName(), nvinfer1::OptProfileSelector::kMIN, - GetTRTBlobDimension(kMINBatchSize, inputDim.channels, inputDim.height, - inputDim.width, params.inputLayout)); - profile->setDimensions(input->getName(), nvinfer1::OptProfileSelector::kOPT, - GetTRTBlobDimension(params.batchSize, inputDim.channels, inputDim.height, - inputDim.width, params.inputLayout)); - profile->setDimensions(input->getName(), nvinfer1::OptProfileSelector::kMAX, - GetTRTBlobDimension(kMAXBatchSize, inputDim.channels, inputDim.height, - inputDim.width, params.inputLayout)); - } - } - return hasDynamicShape; -} - -nvinfer1::DataType GetTRTDataType(TRTBackendPrecision precision) -{ - nvinfer1::DataType dataType; - switch (precision) - { - case TRTBackendPrecision::INT8: - { - dataType = nvinfer1::DataType::kINT8; - break; - } - case TRTBackendPrecision::FP16: - { - dataType = nvinfer1::DataType::kHALF; - break; - } - case TRTBackendPrecision::FP32: - { - dataType = nvinfer1::DataType::kFLOAT; - break; - } - default: - { - dataType = nvinfer1::DataType::kFLOAT; - break; - } - } - return dataType; -} - -nvuffparser::UffInputOrder GetTRTInputOrder(TRTBackendBlobLayout layout) -{ - nvuffparser::UffInputOrder order; - switch (layout) - { - case TRTBackendBlobLayout::PLANAR: - { - order = nvuffparser::UffInputOrder::kNCHW; - break; - } - case TRTBackendBlobLayout::INTERLEAVED: - { - order = nvuffparser::UffInputOrder::kNHWC; - break; - } - default: - { - throw std::runtime_error("Only PLANAR and INTERLEAVED types allowed"); - } - } - return order; -} - -} // anonymous namespace - -class TRTLogger : public nvinfer1::ILogger -{ - -public: - nvinfer1::ILogger &getLogger() - { - return *this; - } - - void log(nvinfer1::ILogger::Severity severity, const char *msg) noexcept override - { - switch (severity) - { - case nvinfer1::ILogger::Severity::kINTERNAL_ERROR: - { - std::cout << msg << std::endl; - break; - } - case nvinfer1::ILogger::Severity::kERROR: - { - std::cout << msg << std::endl; - break; - } - case nvinfer1::ILogger::Severity::kWARNING: - { - std::cout << msg << std::endl; - break; - } - case nvinfer1::ILogger::Severity::kINFO: - { - std::cout << msg << std::endl; - break; - } - default: - { - std::cout << msg << std::endl; - break; - } - } - } -}; - -struct TRTImpl -{ - TRTImpl() - : m_logger(new TRTLogger()) - , m_TRTRuntime(nullptr, [](nvinfer1::IRuntime *runtime) { runtime->destroy(); }) - , m_inferenceEngine(nullptr) - , m_ownedInferenceEngine(nullptr, [](nvinfer1::ICudaEngine *eng) { eng->destroy(); }) - , m_inferContext(nullptr, [](nvinfer1::IExecutionContext *ectx) { ectx->destroy(); }) - , m_cudaStream(0) - { - } - - std::unique_ptr m_logger; - std::unique_ptr m_TRTRuntime; - nvinfer1::ICudaEngine *m_inferenceEngine; - std::unique_ptr m_ownedInferenceEngine; - std::unique_ptr m_inferContext; - - cudaStream_t m_cudaStream; - int m_bindingsCount = 0; - int m_batchSize = 1; - bool m_explicitBatch = false; - TRTBackendPrecision m_precision = TRTBackendPrecision::FP32; - std::unordered_map m_blobMap; - - void loadNetWorkFromFile(const char *modelFilePath); - void loadNetWorkFromUff(TRTBackendParams ¶ms); - void loadNetWorkFromOnnx(TRTBackendParams ¶ms); - void loadFromMemoryPointer(void *engine); - // find the input/output bindings - void setupIO(int batchSize); -}; - -void TRTImpl::loadNetWorkFromFile(const char *modelFilePath) -{ - // Initialize TRT engine and deserialize it from file - std::ifstream trtModelFStream(modelFilePath, std::ios::binary); - std::unique_ptr trtModelContent; - size_t trtModelContentSize = 0; - if (!trtModelFStream.good()) - { - std::cerr << "Model File: " << modelFilePath << std::endl; - throw std::runtime_error("TensorRT: Model file not found."); - } - else - { - trtModelFStream.seekg(0, trtModelFStream.end); - trtModelContentSize = trtModelFStream.tellg(); - trtModelFStream.seekg(0, trtModelFStream.beg); - trtModelContent.reset(new char[trtModelContentSize]); - trtModelFStream.read(trtModelContent.get(), trtModelContentSize); - trtModelFStream.close(); - std::cout << "Deserializing engine from: " << modelFilePath; - } - m_TRTRuntime.reset(nvinfer1::createInferRuntime(*(m_logger.get()))); - m_inferenceEngine = dynamic_cast( - m_TRTRuntime->deserializeCudaEngine(trtModelContent.get(), trtModelContentSize, nullptr)); - m_ownedInferenceEngine.reset(m_inferenceEngine); - m_inferContext.reset(m_inferenceEngine->createExecutionContext()); - m_inferContext->setOptimizationProfile(0); -} - -void TRTImpl::loadNetWorkFromOnnx(TRTBackendParams ¶ms) -{ - if (!params.explicitBatch) - { - std::cerr << "ONNX model only supports explicit batch size"; - } - std::ifstream f(params.enginePath); - if (f.good()) - { - loadNetWorkFromFile(params.enginePath.c_str()); - return; - } - auto builder = nvinfer1::createInferBuilder(*(m_logger.get())); - auto config = builder->createBuilderConfig(); - auto batchFlag = 1U << static_cast(nvinfer1::NetworkDefinitionCreationFlag::kEXPLICIT_BATCH); - auto network = builder->createNetworkV2(batchFlag); - auto parser = nvonnxparser::createParser(*network, *(m_logger.get())); - - // Force FP32 - if (!builder->platformHasFastFp16()) - { - m_precision = TRTBackendPrecision::FP32; - } - - // Configuration - builder->setMaxBatchSize(params.batchSize); - if (!parser->parseFromFile(params.weightsPath.c_str(), 0)) - { - std::cerr << "Fail to parse"; - } - config->setMaxWorkspaceSize(1 << 30); - if (m_precision == TRTBackendPrecision::FP16) - { - config->setFlag(nvinfer1::BuilderFlag::kFP16); - } - - auto profile = builder->createOptimizationProfile(); - if (SetupProfile(profile, network, params)) - { - config->addOptimizationProfile(profile); - } - - // Build the engine - m_inferenceEngine = builder->buildEngineWithConfig(*network, *config); - if (m_inferenceEngine == nullptr) - { - throw std::runtime_error("TensorRT: unable to create engine"); - } - - m_ownedInferenceEngine.reset(m_inferenceEngine); - m_inferContext.reset(m_inferenceEngine->createExecutionContext()); - - network->destroy(); - builder->destroy(); - config->destroy(); - - auto serializedEngine = m_inferenceEngine->serialize(); - WriteSerializedEngineToFile(static_cast(serializedEngine->data()), serializedEngine->size(), - params.enginePath); -} - -void TRTImpl::loadNetWorkFromUff(TRTBackendParams ¶ms) -{ - auto builder = nvinfer1::createInferBuilder(*(m_logger.get())); - auto config = builder->createBuilderConfig(); - auto batchFlag = params.explicitBatch - ? 1U << static_cast(nvinfer1::NetworkDefinitionCreationFlag::kEXPLICIT_BATCH) - : 0U; - auto network = builder->createNetworkV2(batchFlag); - auto parser = nvuffparser::createUffParser(); - - std::ifstream f(params.enginePath); - if (f.good()) - { - loadNetWorkFromFile(params.enginePath.c_str()); - return; - } - nvinfer1::DataType dataType = GetTRTDataType(params.precision); - - // Force FP32 - if (!builder->platformHasFastFp16()) - { - dataType = nvinfer1::DataType::kFLOAT; - m_precision = TRTBackendPrecision::FP32; - } - - // Register uff input - for (int i = 0; i < params.inputLayers.size(); i++) - { - if (params.explicitBatch) - { - parser->registerInput( - params.inputLayers[i].c_str(), - GetTRTBlobDimension(params.batchSize, params.inputDims[i].channels, params.inputDims[i].height, - params.inputDims[i].width, params.inputLayout), - GetTRTInputOrder(params.inputLayout)); - } - else - { - parser->registerInput(params.inputLayers[i].c_str(), - GetTRTBlobDimension(params.inputDims[i].channels, params.inputDims[i].height, - params.inputDims[i].width, params.inputLayout), - GetTRTInputOrder(params.inputLayout)); - } - } - - // Register uff output - for (int i = 0; i < params.outputLayers.size(); i++) - { - parser->registerOutput(params.outputLayers[i].c_str()); - } - - // Parse uff model - if (!parser->parse(params.weightsPath.c_str(), *network, dataType)) - { - std::cerr << "Fail to parse"; - } - - // Configuration - if (params.explicitBatch) - { - auto profile = builder->createOptimizationProfile(); - if (SetupProfile(profile, network, params)) - { - config->addOptimizationProfile(profile); - } - } - else - { - builder->setMaxBatchSize(params.batchSize); - } - - config->setMaxWorkspaceSize(1 << 30); - if (m_precision == TRTBackendPrecision::FP16) - { - config->setFlag(nvinfer1::BuilderFlag::kFP16); - } - - // Build the engine - m_inferenceEngine = builder->buildEngineWithConfig(*network, *config); - if (m_inferenceEngine == nullptr) - { - throw std::runtime_error("TensorRT: unable to create engine"); - } - - m_ownedInferenceEngine.reset(m_inferenceEngine); - m_inferContext.reset(m_inferenceEngine->createExecutionContext()); - - network->destroy(); - builder->destroy(); - config->destroy(); - - auto serializedEngine = m_inferenceEngine->serialize(); - WriteSerializedEngineToFile(static_cast(serializedEngine->data()), serializedEngine->size(), - params.enginePath); -} - -void TRTImpl::loadFromMemoryPointer(void *engine) -{ - m_inferenceEngine = static_cast(engine); - m_inferContext.reset(m_inferenceEngine->createExecutionContext()); -} - -void TRTImpl::setupIO(int batchSize) -{ - // @TODO: use getBindingDimensions to avoid re-setting the IO. - m_bindingsCount = m_inferenceEngine->getNbBindings(); - for (int i = 0; i < m_bindingsCount; i++) - { - m_blobMap[std::string(m_inferenceEngine->getBindingName(i))] = i; - if (m_inferenceEngine->bindingIsInput(i)) - { - nvinfer1::Dims dims_i(m_inferenceEngine->getBindingDimensions(i)); - nvinfer1::Dims4 inputDims{batchSize, dims_i.d[1], dims_i.d[2], dims_i.d[3]}; - m_inferContext->setBindingDimensions(i, inputDims); - } - } -} - -TRTBackend::TRTBackend(const char *modelFilePath, TRTBackendPrecision precision, int batchSize, bool explicitBatch) - : m_pImpl(new TRTImpl()) -{ - m_pImpl->m_precision = precision; - m_pImpl->m_batchSize = batchSize; - m_pImpl->m_explicitBatch = explicitBatch; - m_pImpl->loadNetWorkFromFile(modelFilePath); - m_pImpl->setupIO(m_pImpl->m_batchSize); -} - -TRTBackend::TRTBackend(TRTBackendParams &inputParams) - : m_pImpl(new TRTImpl()) -{ - m_pImpl->m_precision = inputParams.precision; - m_pImpl->m_batchSize = inputParams.batchSize; - m_pImpl->m_explicitBatch = inputParams.explicitBatch; - switch (inputParams.modelType) - { - case ModelType::ONNX: - { - m_pImpl->loadNetWorkFromOnnx(inputParams); - break; - } - case ModelType::UFF: - { - m_pImpl->loadNetWorkFromUff(inputParams); - break; - } - case ModelType::TRT_ENGINE: - { - m_pImpl->loadNetWorkFromFile(inputParams.enginePath.c_str()); - break; - } - case ModelType::TRT_ENGINE_IN_MEMORY: - { - m_pImpl->loadFromMemoryPointer(inputParams.trtEngineInMemory); - break; - } - default: - { - throw std::runtime_error( - "Only Model types ONNX, UFF, TensorRT " - "serialized engines and a pointer to deserialized " - "ICudaEngine are supported\n"); - } - } - m_pImpl->setupIO(m_pImpl->m_batchSize); -} - -TRTBackend::~TRTBackend() {} - -void TRTBackend::infer(void **buffer) -{ - bool success = true; - if (!m_pImpl->m_inferenceEngine->hasImplicitBatchDimension()) - { - m_pImpl->m_inferContext->enqueueV2(buffer, m_pImpl->m_cudaStream, nullptr); - } - else - { - m_pImpl->m_inferContext->enqueue(m_pImpl->m_batchSize, buffer, m_pImpl->m_cudaStream, nullptr); - } - - if (!success) - { - throw std::runtime_error("TensorRT: Inference failed"); - } -} - -void TRTBackend::infer(void **buffer, int batchSize, cudaStream_t stream) -{ - //@TODO: fix kMin, kOpt, kMax batch size in SetupProfile() call and then add a check here. - m_pImpl->setupIO(batchSize); - - bool success = true; - if (!m_pImpl->m_inferenceEngine->hasImplicitBatchDimension()) - { - m_pImpl->m_inferContext->enqueueV2(buffer, stream, nullptr); - } - else - { - m_pImpl->m_inferContext->enqueue(batchSize, buffer, stream, nullptr); - } - - if (!success) - { - throw std::runtime_error("TensorRT: Inference failed"); - } -} - -cudaStream_t TRTBackend::getCUDAStream() const -{ - return m_pImpl->m_cudaStream; -} - -void TRTBackend::setCUDAStream(cudaStream_t stream) -{ - m_pImpl->m_cudaStream = stream; -} - -int TRTBackend::getBlobCount() const -{ - return m_pImpl->m_bindingsCount; -} - -TRTBackendBlobSize TRTBackend::getTRTBackendBlobSize(int blobIndex) const -{ - if (blobIndex >= m_pImpl->m_bindingsCount) - { - throw std::runtime_error("blobIndex out of range"); - } - auto dim = m_pImpl->m_inferenceEngine->getBindingDimensions(blobIndex); - TRTBackendBlobSize blobSize; - if (dim.nbDims == 2) - { - blobSize = {1, dim.d[0], dim.d[1]}; - } - else if (dim.nbDims == 3) - { - blobSize = {dim.d[0], dim.d[1], dim.d[2]}; - } - else if (dim.nbDims == 4) - { - blobSize = {dim.d[1], dim.d[2], dim.d[3]}; - } - else - { - throw std::runtime_error("Unknown TensorRT binding dimension!"); - } - return blobSize; -} - -int TRTBackend::getBlobLinearSize(int blobIndex) const -{ - const TRTBackendBlobSize shape = getTRTBackendBlobSize(blobIndex); - nvinfer1::Dims3 dims_val{shape.channels, shape.height, shape.width}; - int blobSize = 1; - for (int i = 0; i < 3; i++) - { - blobSize *= dims_val.d[i] <= 0 ? 1 : dims_val.d[i]; - } - return blobSize; -} - -int TRTBackend::getBlobIndex(const char *blobName) const -{ - auto blobItr = m_pImpl->m_blobMap.find(std::string(blobName)); - if (blobItr == m_pImpl->m_blobMap.end()) - { - throw std::runtime_error("blobName not found"); - } - return blobItr->second; -} - -bool TRTBackend::bindingIsInput(const int index) const -{ - return m_pImpl->m_inferenceEngine->bindingIsInput(index); -} - -} // namespace cvcore diff --git a/isaac_ros_bi3d/gxf/bi3d/extensions/bi3d/Bi3D.cpp b/isaac_ros_bi3d/gxf/bi3d/extensions/bi3d/Bi3D.cpp deleted file mode 100644 index 1389789..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/extensions/bi3d/Bi3D.cpp +++ /dev/null @@ -1,476 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -#include - -#include "extensions/bi3d/Bi3D.hpp" - -#include "gxf/cuda/cuda_stream_pool.hpp" -#include "gxf/cuda/cuda_stream_id.hpp" -#include "gxf/multimedia/video.hpp" -#include "gxf/std/tensor.hpp" -#include "gxf/std/timestamp.hpp" - -namespace nvidia { -namespace cvcore { - -namespace detail { - -// Function to bind a cuda stream with cid into downstream message -gxf_result_t BindCudaStream(gxf::Entity& entity, gxf_uid_t cid) { - if (cid == kNullUid) { - GXF_LOG_ERROR("stream_cid is null"); - return GXF_FAILURE; - } - auto output_stream_id = entity.add("stream"); - if (!output_stream_id) { - GXF_LOG_ERROR("failed to add cudastreamid."); - return output_stream_id.error(); - } - output_stream_id.value()->stream_cid = cid; - return GXF_SUCCESS; -} - -// Function to record a new cuda event -gxf_result_t BindCudaEvent(gxf::Entity & entity, - gxf::Handle & stream) { - - cudaEvent_t event_id; - cudaEventCreateWithFlags(&event_id, 0); - gxf::CudaEvent event; - event.initWithEvent(event_id, stream->dev_id(), [](auto){}); - - auto ret = stream->record(event.event().value(), - [event = event_id, entity = entity.clone().value()](auto) { - cudaEventDestroy(event); - }); - if (!ret) { - GXF_LOG_ERROR("record event failed"); - return ret.error(); - } - - return GXF_SUCCESS; -} - -} // namespace detail - -gxf_result_t Bi3D::registerInterface(gxf::Registrar* registrar) { - gxf::Expected result; - - result &= registrar->parameter(left_image_name_, "left_image_name", "The name of the left image to be received", "", - gxf::Registrar::NoDefaultParameter(), GXF_PARAMETER_FLAGS_OPTIONAL); - result &= - registrar->parameter(right_image_name_, "right_image_name", "The name of the right image to be received", "", - gxf::Registrar::NoDefaultParameter(), GXF_PARAMETER_FLAGS_OPTIONAL); - result &= registrar->parameter(output_name_, "output_name", - "The name of the tensor to be passed to next node", ""); - result &= registrar->parameter(stream_pool_, "stream_pool", "cuda stream pool", "cuda stream pool object", - gxf::Registrar::NoDefaultParameter(), GXF_PARAMETER_FLAGS_OPTIONAL); - result &= registrar->parameter(pool_, "pool", "Memory pool for allocating output data", ""); - result &= registrar->parameter(left_image_receiver_, "left_image_receiver", - "Receiver to get the left image", ""); - result &= registrar->parameter(right_image_receiver_, "right_image_receiver", - "Receiver to get the right image", ""); - result &= registrar->parameter(disparity_receiver_, "disparity_receiver", - "Optional receilver for dynamic disparity input", "", - gxf::Registrar::NoDefaultParameter(), GXF_PARAMETER_FLAGS_OPTIONAL); - result &= registrar->parameter(output_transmitter_, "output_transmitter", "Transmitter to send the data", - ""); - - result &= registrar->parameter(image_type_, "image_type", "Type of input image: BGR_U8 or RGB_U8", "", - gxf::Registrar::NoDefaultParameter(), GXF_PARAMETER_FLAGS_OPTIONAL); - result &= registrar->parameter(pixel_mean_, "pixel_mean", "The mean for each channel", ""); - result &= registrar->parameter(normalization_, "normalization", "The normalization for each channel", ""); - result &= - registrar->parameter(standard_deviation_, "standard_deviation", - "The standard deviation for each channel", ""); - - result &= registrar->parameter(max_batch_size_, "max_batch_size", "The max batch size to run inference on", - ""); - result &= registrar->parameter(input_layer_width_, "input_layer_width", "The model input layer width", ""); - result &= registrar->parameter(input_layer_height_, "input_layer_height", "The model input layer height", - ""); - result &= registrar->parameter(model_input_type_, "model_input_type", - "The model input image: BGR_U8 or RGB_U8", "", - gxf::Registrar::NoDefaultParameter(), GXF_PARAMETER_FLAGS_OPTIONAL); - - result &= registrar->parameter(featnet_engine_file_path_, "featnet_engine_file_path", - "The path to the serialized TRT engine of featnet", ""); - result &= - registrar->parameter(featnet_input_layers_name_, "featnet_input_layers_name", - "The names of the input layers", ""); - result &= registrar->parameter(featnet_output_layers_name_, "featnet_output_layers_name", - "The names of the output layers", ""); - - result &= registrar->parameter(featnet_hr_engine_file_path_, "featnet_hr_engine_file_path", - "The path to the serialized TRT engine of high-resolution featnet", "", - gxf::Registrar::NoDefaultParameter(), GXF_PARAMETER_FLAGS_OPTIONAL); - result &= registrar->parameter(featnet_hr_input_layers_name_, "featnet_hr_input_layers_name", - "The names of the input layers", "", - gxf::Registrar::NoDefaultParameter(), GXF_PARAMETER_FLAGS_OPTIONAL); - result &= registrar->parameter(featnet_hr_output_layers_name_, "featnet_hr_output_layers_name", - "The names of the output layers", "", - gxf::Registrar::NoDefaultParameter(), GXF_PARAMETER_FLAGS_OPTIONAL); - - result &= registrar->parameter(refinenet_engine_file_path_, "refinenet_engine_file_path", - "The path to the serialized TRT engine of refinenet", "", - gxf::Registrar::NoDefaultParameter(), GXF_PARAMETER_FLAGS_OPTIONAL); - result &= registrar->parameter(refinenet_input_layers_name_, "refinenet_input_layers_name", - "The names of the input layers", "", - gxf::Registrar::NoDefaultParameter(), GXF_PARAMETER_FLAGS_OPTIONAL); - result &= registrar->parameter(refinenet_output_layers_name_, "refinenet_output_layers_name", - "The names of the output layers", "", - gxf::Registrar::NoDefaultParameter(), GXF_PARAMETER_FLAGS_OPTIONAL); - - result &= registrar->parameter(segnet_engine_file_path_, "segnet_engine_file_path", - "The path to the serialized TRT engine of segnet", ""); - result &= - registrar->parameter(segnet_input_layers_name_, "segnet_input_layers_name", - "The names of the input layers", ""); - result &= - registrar->parameter(segnet_output_layers_name_, "segnet_output_layers_name", - "The names of the output layers", ""); - - result &= registrar->parameter(engine_type_, "engine_type", "The type of engine to be run", ""); - result &= registrar->parameter(apply_sigmoid_, "apply_sigmoid", "Whether to apply sigmoid operation", ""); - result &= registrar->parameter(apply_thresholding_, "apply_thresholding", - "Whether to apply threshold operation", ""); - result &= registrar->parameter(threshold_value_low_, "threshold_value_low", - "Low value set by thresholding operation", ""); - result &= registrar->parameter(threshold_value_high_, "threshold_value_high", - "High value set by thresholding operation", ""); - result &= registrar->parameter(threshold_, "threshold", - "Pixel value used by thresholding operation that casts to low or high", ""); - result &= - registrar->parameter(apply_edge_refinement_, "apply_edge_refinement", - "Whether to apply edge refinement", "", false); - - result &= registrar->parameter(max_disparity_levels_, "max_disparity_levels", - "The maximum number of output disparity levels", ""); - result &= registrar->parameter(disparity_values_, "disparity_values", "Input disparity values array", "", - gxf::Registrar::NoDefaultParameter(), GXF_PARAMETER_FLAGS_OPTIONAL); - result &= registrar->parameter(timestamp_policy_, "timestamp_policy", - "Input channel to get timestamp 0(left)/1(right)", "", 0); - - return gxf::ToResultCode(result); -} - -gxf_result_t Bi3D::start() { - - // Allocate cuda stream using stream pool if necessary - if (stream_pool_.try_get()) { - auto stream = stream_pool_.try_get().value()->allocateStream(); - if (!stream) { - GXF_LOG_ERROR("allocating stream failed."); - return GXF_FAILURE; - } - cuda_stream_ = std::move(stream.value()); - if (!cuda_stream_->stream()) { - GXF_LOG_ERROR("allocated stream is not initialized."); - return GXF_FAILURE; - } - } - - // Setting image pre-processing params for Bi3D - const auto& pixel_mean_vec = pixel_mean_.get(); - const auto& normalization_vec = normalization_.get(); - const auto& standard_deviation_vec = standard_deviation_.get(); - if (pixel_mean_vec.size() != 3 || normalization_vec.size() != 3 || standard_deviation_vec.size() != 3) { - GXF_LOG_ERROR("Invalid preprocessing params."); - return GXF_FAILURE; - } - std::copy(pixel_mean_vec.begin(), pixel_mean_vec.end(), preProcessorParams.pixelMean); - std::copy(normalization_vec.begin(), normalization_vec.end(), preProcessorParams.normalization); - std::copy(standard_deviation_vec.begin(), standard_deviation_vec.end(), preProcessorParams.stdDev); - - // Setting model input params for Bi3D - modelInputParams.maxBatchSize = max_batch_size_.get(); - modelInputParams.inputLayerWidth = input_layer_width_.get(); - modelInputParams.inputLayerHeight = input_layer_height_.get(); - - // Setting inference params for Bi3D - auto toComputeEngine = [](std::string type) -> gxf::Expected { - if (type == "GPU") { - return -1; - } else if (type == "DLA_CORE_0") { - return 0; - } else if (type == "DLA_CORE_1") { - return 1; - } else { - GXF_LOG_ERROR("Invalid compute engine type."); - return gxf::Unexpected{GXF_FAILURE}; - } - }; - auto engineType = toComputeEngine(engine_type_.get()); - if (!engineType) { - return engineType.error(); - } - featureInferenceParams = ::cvcore::inferencer::TensorRTInferenceParams{ - ::cvcore::inferencer::TRTInferenceType::TRT_ENGINE, - nullptr, - featnet_engine_file_path_.get(), - 1, - featnet_input_layers_name_.get(), - featnet_output_layers_name_.get(), - engineType.value()}; - featureHRInferenceParams = ::cvcore::inferencer::TensorRTInferenceParams{ - ::cvcore::inferencer::TRTInferenceType::TRT_ENGINE, - nullptr, - featnet_hr_engine_file_path_.try_get().value_or(""), - 1, - featnet_hr_input_layers_name_.try_get().value_or(std::vector{""}), - featnet_hr_output_layers_name_.try_get().value_or(std::vector{""}), - engineType.value()}; - refinementInferenceParams = ::cvcore::inferencer::TensorRTInferenceParams{ - ::cvcore::inferencer::TRTInferenceType::TRT_ENGINE, - nullptr, - refinenet_engine_file_path_.try_get().value_or(""), - 1, - refinenet_input_layers_name_.try_get().value_or(std::vector{""}), - refinenet_output_layers_name_.try_get().value_or(std::vector{""}), - engineType.value()}; - segmentationInferenceParams = ::cvcore::inferencer::TensorRTInferenceParams{ - ::cvcore::inferencer::TRTInferenceType::TRT_ENGINE, - nullptr, - segnet_engine_file_path_.get(), - 1, - segnet_input_layers_name_.get(), - segnet_output_layers_name_.get(), - engineType.value()}; - - // Setting extra params for Bi3D - auto toProcessingControl = [](bool flag) { - return flag ? ::cvcore::bi3d::ProcessingControl::ENABLE : ::cvcore::bi3d::ProcessingControl::DISABLE; - }; - extraParams = {max_disparity_levels_.get(), - toProcessingControl(apply_edge_refinement_.get()), - toProcessingControl(apply_sigmoid_.get()), - toProcessingControl(apply_thresholding_.get()), - threshold_value_low_.get(), - threshold_value_high_.get(), - threshold_.get()}; - - // Setting Bi3D object with the provided params - objBi3D.reset(new ::cvcore::bi3d::Bi3D(preProcessorParams, modelInputParams, featureInferenceParams, - featureHRInferenceParams, refinementInferenceParams, - segmentationInferenceParams, extraParams)); - - const auto& disparity_values_vec = disparity_values_.try_get(); - if (disparity_values_vec) { - if (disparity_values_vec.value().empty() || disparity_values_vec.value().size() > max_disparity_levels_.get()) { - GXF_LOG_ERROR("Invalid disparity values."); - return GXF_FAILURE; - } - disparityValues = ::cvcore::Array(disparity_values_vec.value().size(), true); - disparityValues.setSize(disparity_values_vec.value().size()); - - for (size_t i = 0; i < disparity_values_vec.value().size(); i++) { - disparityValues[i] = disparity_values_vec.value()[i]; - } - } - return GXF_SUCCESS; -} - -gxf_result_t Bi3D::tick() { - - // Get a CUDA stream for execution - cudaStream_t cuda_stream = 0; - if(cuda_stream_.try_get()) - { - cuda_stream = cuda_stream_->stream().value(); - } - - // Receiving the data synchronously across streams - auto inputLeftMessage = left_image_receiver_->receive(); - if (!inputLeftMessage) { - return inputLeftMessage.error(); - } else if (cuda_stream != 0) { - detail::BindCudaEvent(inputLeftMessage.value(), cuda_stream_); - auto inputLeft_stream_id = inputLeftMessage.value().get("stream"); - if(inputLeft_stream_id) { - auto inputLeft_stream = gxf::Handle::Create(inputLeft_stream_id.value().context(), - inputLeft_stream_id.value()->stream_cid); - // NOTE: This is an expensive call. It will halt the current CPU thread until all events - // previously associated with the stream are cleared - inputLeft_stream.value()->syncStream(); - } - } - auto inputRightMessage = right_image_receiver_->receive(); - if (!inputRightMessage) { - return inputRightMessage.error(); - } else if (cuda_stream != 0) { - detail::BindCudaEvent(inputRightMessage.value(), cuda_stream_); - auto inputRight_stream_id = inputRightMessage.value().get("stream"); - if(inputRight_stream_id) { - auto inputRight_stream = gxf::Handle::Create(inputRight_stream_id.value().context(), - inputRight_stream_id.value()->stream_cid); - // NOTE: This is an expensive call. It will halt the current CPU thread until all events - // previously associated with the stream are cleared - inputRight_stream.value()->syncStream(); - } - } - - auto maybeLeftName = left_image_name_.try_get(); - auto inputLeftBuffer = inputLeftMessage.value().get( - maybeLeftName ? maybeLeftName.value().c_str() : nullptr); - if (!inputLeftBuffer) { - return inputLeftBuffer.error(); - } - auto maybeRightName = right_image_name_.try_get(); - auto inputRightBuffer = inputRightMessage.value().get( - maybeRightName ? maybeRightName.value().c_str() : nullptr); - if (!inputRightBuffer) { - return inputRightBuffer.error(); - } - if (inputLeftBuffer.value()->storage_type() != gxf::MemoryStorageType::kDevice || - inputRightBuffer.value()->storage_type() != gxf::MemoryStorageType::kDevice) { - GXF_LOG_ERROR("input images must be on GPU."); - return GXF_FAILURE; - } - const auto & left_info = inputLeftBuffer.value()->video_frame_info(); - const auto & right_info = inputRightBuffer.value()->video_frame_info(); - if (left_info.color_format != right_info.color_format || - left_info.height != right_info.height || left_info.width != right_info.width || - left_info.surface_layout != left_info.surface_layout || - (left_info.color_format != gxf::VideoFormat::GXF_VIDEO_FORMAT_RGB && - left_info.color_format != gxf::VideoFormat::GXF_VIDEO_FORMAT_R32_G32_B32 && - left_info.color_format != gxf::VideoFormat::GXF_VIDEO_FORMAT_BGR && - left_info.color_format != gxf::VideoFormat::GXF_VIDEO_FORMAT_B32_G32_R32)) { - return GXF_INVALID_DATA_FORMAT; - } - - // Create output message - gxf::Expected outputMessage = gxf::Entity::New(context()); - if (!outputMessage) { - GXF_LOG_ERROR("Bi3D::tick ==> Failed to create output message."); - return outputMessage.error(); - } - - // Receive the disparity array if necessary - const auto& maybeDisparityReceiver = disparity_receiver_.try_get(); - if (maybeDisparityReceiver) { - auto disparityMessage = maybeDisparityReceiver.value()->receive(); - if (!disparityMessage) { - return disparityMessage.error(); - } - auto disparityTensor = disparityMessage.value().get(); - if (!disparityTensor) { - return disparityTensor.error(); - } - if (disparityTensor.value()->element_count() > max_disparity_levels_.get() || - disparityTensor.value()->element_type() != gxf::PrimitiveType::kInt32 || - disparityTensor.value()->storage_type() == gxf::MemoryStorageType::kDevice) { - GXF_LOG_ERROR("invalid input disparity values."); - return GXF_FAILURE; - } - const auto disparityCount = disparityTensor.value()->element_count(); - disparityValues = ::cvcore::Array(disparityCount, true); - disparityValues.setSize(disparityCount); - - for (size_t i = 0; i < disparityCount; i++) { - disparityValues[i] = disparityTensor.value()->data().value()[i]; - } - - auto forwardedDisparity = outputMessage.value().add(disparityTensor->name()); - if (strcmp(disparityTensor->name(), output_name_.get().c_str()) == 0) { - GXF_LOG_ERROR("Bi3D::tick ==> Forwarded disparity array name and network output name must differ."); - return GXF_FAILURE; - } - if (!forwardedDisparity) { - GXF_LOG_ERROR("Bi3D::tick ==> Failed to forward input disparity tensor in output message."); - return forwardedDisparity.error(); - } - *forwardedDisparity.value() = std::move(*disparityTensor.value()); - } - - // Creating GXF tensor to hold the data to be transmitted - auto outputImage = outputMessage.value().add(output_name_.get().c_str()); - if (!outputImage) { - GXF_LOG_ERROR("Bi3D::tick ==> Failed to create tensor in output message."); - return outputImage.error(); - } - auto result = outputImage.value()->reshape({static_cast(disparityValues.getSize()), - static_cast(left_info.height), - static_cast(left_info.width)}, - gxf::MemoryStorageType::kDevice, pool_); - if (!result) { - GXF_LOG_ERROR("Bi3D::tick ==> Failed to allocate tensor in output message."); - return result.error(); - } - - // Creating CVCore Tensors to hold the input and output data - ::cvcore::Tensor<::cvcore::CHW, ::cvcore::CX, ::cvcore::F32> outputImageDevice( - left_info.width, left_info.height, disparityValues.getSize(), - const_cast(outputImage.value()->data().value()), false); - - // Running the inference - if (left_info.color_format == gxf::VideoFormat::GXF_VIDEO_FORMAT_RGB || - left_info.color_format == gxf::VideoFormat::GXF_VIDEO_FORMAT_BGR) { - ::cvcore::Tensor<::cvcore::HWC, ::cvcore::C3, ::cvcore::U8> leftImageDevice( - left_info.width, left_info.height, - reinterpret_cast(inputLeftBuffer.value()->pointer()), false); - ::cvcore::Tensor<::cvcore::HWC, ::cvcore::C3, ::cvcore::U8> rightImageDevice( - right_info.width, right_info.height, - reinterpret_cast(inputRightBuffer.value()->pointer()), false); - objBi3D->execute(outputImageDevice, - leftImageDevice, rightImageDevice, - disparityValues, cuda_stream); - } else if (left_info.color_format == gxf::VideoFormat::GXF_VIDEO_FORMAT_R32_G32_B32 || - left_info.color_format == gxf::VideoFormat::GXF_VIDEO_FORMAT_B32_G32_R32) { - ::cvcore::Tensor<::cvcore::CHW, ::cvcore::C3, ::cvcore::F32> leftImageDevice( - left_info.width, left_info.height, - reinterpret_cast(inputLeftBuffer.value()->pointer()), false); - ::cvcore::Tensor<::cvcore::CHW, ::cvcore::C3, ::cvcore::F32> rightImageDevice( - right_info.width, right_info.height, - reinterpret_cast(inputRightBuffer.value()->pointer()), false); - objBi3D->execute(outputImageDevice, - leftImageDevice, rightImageDevice, - disparityValues, cuda_stream); - } else { - return GXF_FAILURE; - } - - // Allocate a cuda event that can be used to record on each tick - if(cuda_stream_.try_get()) - { - detail::BindCudaStream(outputMessage.value(), cuda_stream_.cid()); - detail::BindCudaEvent(outputMessage.value(), cuda_stream_); - } - - // Pass down timestamp if necessary - auto maybeDaqTimestamp = timestamp_policy_.get() == 0 ? inputLeftMessage.value().get() - : inputRightMessage.value().get(); - if (maybeDaqTimestamp) { - auto outputTimestamp = outputMessage.value().add(maybeDaqTimestamp.value().name()); - if (!outputTimestamp) { - return outputTimestamp.error(); - } - *outputTimestamp.value() = *maybeDaqTimestamp.value(); - } - - // Send the data - output_transmitter_->publish(outputMessage.value()); - return GXF_SUCCESS; -} - -gxf_result_t Bi3D::stop() { - objBi3D.reset(nullptr); - return GXF_SUCCESS; -} - -} // namespace cvcore -} // namespace nvidia diff --git a/isaac_ros_bi3d/gxf/bi3d/extensions/bi3d/Bi3DRegistry.cpp b/isaac_ros_bi3d/gxf/bi3d/extensions/bi3d/Bi3DRegistry.cpp deleted file mode 100644 index cb356c6..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/extensions/bi3d/Bi3DRegistry.cpp +++ /dev/null @@ -1,27 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -#include "extensions/bi3d/Bi3D.hpp" -#include "gxf/std/extension_factory_helper.hpp" - -GXF_EXT_FACTORY_BEGIN() -GXF_EXT_FACTORY_SET_INFO(0x62a390bbf9f842fb, 0xba93f93393a27d4a, "NvCvBi3DExtension", "CVCORE Bi3D module", - "Nvidia_Gxf", "1.0.1", "LICENSE"); - -GXF_EXT_FACTORY_ADD(0xdcba0cf83a5340d2, 0x8404788350ad2324, nvidia::cvcore::Bi3D, nvidia::gxf::Codelet, - "Bi3D GXF Extension"); -GXF_EXT_FACTORY_END() diff --git a/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/CameraModel.cpp b/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/CameraModel.cpp deleted file mode 100644 index 5dce98e..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/CameraModel.cpp +++ /dev/null @@ -1,86 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -#include "CameraModel.hpp" - -namespace nvidia { -namespace cvcore { -namespace tensor_ops { - -namespace detail { - -gxf::Expected<::cvcore::CameraDistortionType> GetCameraDistortionType(const std::string& type) { - if (type == "Perspective") { - return ::cvcore::CameraDistortionType::NONE; - } else if (type == "Polynomial") { - return ::cvcore::CameraDistortionType::Polynomial; - } else if (type == "FisheyeEquidistant") { - return ::cvcore::CameraDistortionType::FisheyeEquidistant; - } else if (type == "FisheyeEquisolid") { - return ::cvcore::CameraDistortionType::FisheyeEquisolid; - } else if (type == "FisheyeOrthoGraphic") { - return ::cvcore::CameraDistortionType::FisheyeOrthoGraphic; - } else if (type == "FisheyeStereographic") { - return ::cvcore::CameraDistortionType::FisheyeStereographic; - } else { - return gxf::Unexpected{GXF_FAILURE}; - } -} - -} // namespace detail - -gxf_result_t CameraModel::registerInterface(gxf::Registrar* registrar) { - gxf::Expected result; - - result &= registrar->parameter(distortion_type_, "distortion_type"); - result &= registrar->parameter(distortion_coefficients_, "distortion_coefficients"); - result &= registrar->parameter(focal_length_, "focal_length"); - result &= registrar->parameter(principle_point_, "principle_point"); - result &= registrar->parameter(skew_value_, "skew_value"); - - return gxf::ToResultCode(result); -} - -gxf_result_t CameraModel::initialize() { - // Construct distortion model - auto type = detail::GetCameraDistortionType(distortion_type_.get()); - if (!type) { - return GXF_FAILURE; - } - if (distortion_coefficients_.get().size() != 8) { - GXF_LOG_ERROR("size of distortion coefficients must be 8."); - return GXF_FAILURE; - } - for (size_t i = 0; i < 8; i++) { - distortions_.coefficients[i] = distortion_coefficients_.get()[i]; - } - distortions_.type = type.value(); - - // Construct intrinsic model - if (focal_length_.get().size() != 2 || principle_point_.get().size() != 2) { - GXF_LOG_ERROR("focal length and principle point must be 2-element array."); - return GXF_FAILURE; - } - intrinsics_ = ::cvcore::CameraIntrinsics(focal_length_.get()[0], focal_length_.get()[1], principle_point_.get()[0], - principle_point_.get()[1], skew_value_.get()); - - return GXF_SUCCESS; -} - -} // namespace tensor_ops -} // namespace cvcore -} // namespace nvidia diff --git a/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/CameraModel.hpp b/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/CameraModel.hpp deleted file mode 100644 index e68c67f..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/CameraModel.hpp +++ /dev/null @@ -1,60 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 -#ifndef NVIDIA_CVCORE_CAMERA_MODEL_HPP -#define NVIDIA_CVCORE_CAMERA_MODEL_HPP - -#include "gxf/core/component.hpp" -#include "gxf/std/parameter_parser_std.hpp" - -#include "cv/core/CameraModel.h" - -namespace nvidia { -namespace cvcore { -namespace tensor_ops { - -// Wrapper of CameraModel compatible with CVCORE -class CameraModel : public gxf::Component { -public: - virtual ~CameraModel() = default; - CameraModel() = default; - - gxf_result_t registerInterface(gxf::Registrar* registrar) override; - gxf_result_t initialize() override; - - ::cvcore::CameraDistortionModel getDistortionModel() const { - return distortions_; - } - ::cvcore::CameraIntrinsics getCameraIntrinsics() const { - return intrinsics_; - } - -private: - gxf::Parameter distortion_type_; - gxf::Parameter> distortion_coefficients_; - gxf::Parameter> focal_length_; - gxf::Parameter> principle_point_; - gxf::Parameter skew_value_; - - ::cvcore::CameraDistortionModel distortions_; - ::cvcore::CameraIntrinsics intrinsics_; -}; - -} // namespace tensor_ops -} // namespace cvcore -} // namespace nvidia - -#endif diff --git a/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/ConvertColorFormat.cpp b/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/ConvertColorFormat.cpp deleted file mode 100644 index 074e013..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/ConvertColorFormat.cpp +++ /dev/null @@ -1,214 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 -#include "ConvertColorFormat.hpp" - -namespace nvidia { -namespace cvcore { -namespace tensor_ops { - -namespace detail { - -template<::cvcore::ImageType T_IN, ::cvcore::ImageType T_OUT> -gxf_result_t ConvertColorFormatImpl(gxf::Entity& output, gxf::Entity& input, const ImageInfo& output_info, - const ImageInfo& input_info, const char* output_name, const char* input_name, - gxf::Handle output_adapter, gxf::Handle input_adapter, - gxf::Handle allocator, - ::cvcore::tensor_ops::ColorConversionType type, cudaStream_t stream) { - auto input_image = input_adapter->WrapImageFromMessage(input, input_name); - if (!input_image) { - return GXF_FAILURE; - } - - auto error = output_adapter->AddImageToMessage(output, output_info.width, output_info.height, allocator, - output_info.is_cpu, output_name); - if (error != GXF_SUCCESS) { - return GXF_FAILURE; - } - - auto output_image = output_adapter->WrapImageFromMessage(output, output_name); - if (!output_image) { - return GXF_FAILURE; - } - - ::cvcore::tensor_ops::ConvertColorFormat(output_image.value(), input_image.value(), type, stream); - return GXF_SUCCESS; -} - -template<::cvcore::ImageType T_IN, ::cvcore::ImageType T_OUT> -gxf_result_t ConvertColorFormatStreamImpl(gxf::Entity& output, gxf::Entity& input, const ImageInfo& output_info, - const ImageInfo& input_info, const char* output_name, const char* input_name, - gxf::Handle stream, gxf::Handle output_adapter, - gxf::Handle input_adapter, - gxf::Handle allocator) { - auto input_image = input_adapter->WrapImageFromMessage(input, input_name); - if (!input_image) { - return GXF_FAILURE; - } - - auto error = output_adapter->AddImageToMessage(output, output_info.width, output_info.height, allocator, - output_info.is_cpu, output_name); - if (error != GXF_SUCCESS) { - return GXF_FAILURE; - } - - auto output_image = output_adapter->WrapImageFromMessage(output, output_name); - if (!output_image) { - return GXF_FAILURE; - } - - auto err_code = stream->getStream()->ColorConvert(output_image.value(), input_image.value()); - if (err_code != ::cvcore::make_error_condition(::cvcore::ErrorCode::SUCCESS)) { - GXF_LOG_ERROR("color conversion operation failed."); - return GXF_FAILURE; - } - - return GXF_SUCCESS; -} - -} // namespace detail - -template -gxf_result_t ConvertColorFormatBase::registerInterface(gxf::Registrar* registrar) { - gxf::Expected result; - - result &= registrar->parameter(output_type_, "output_type"); - result &= registrar->parameter(receiver_, "receiver"); - result &= registrar->parameter(transmitter_, "transmitter"); - result &= registrar->parameter(pool_, "pool"); - result &= registrar->parameter(stream_, "stream", "tensor stream", "tensor stream object", - gxf::Registrar::NoDefaultParameter(), GXF_PARAMETER_FLAGS_OPTIONAL); - result &= registrar->parameter(stream_pool_, "stream_pool", "cuda stream pool", "cuda stream pool object", - gxf::Registrar::NoDefaultParameter(), GXF_PARAMETER_FLAGS_OPTIONAL); - result &= registrar->parameter(input_adapter_, "input_adapter"); - result &= registrar->parameter(output_adapter_, "output_adapter"); - result &= registrar->parameter(input_name_, "input_name", "input name", "input tensor name", - gxf::Registrar::NoDefaultParameter(), GXF_PARAMETER_FLAGS_OPTIONAL); - result &= registrar->parameter(output_name_, "output_name", "output name", "output tensor name", - gxf::Registrar::NoDefaultParameter(), GXF_PARAMETER_FLAGS_OPTIONAL); - - return gxf::ToResultCode(result); -} - -template -gxf::Expected ConvertColorFormatBase::doInferOutputInfo(gxf::Entity& input) { - // Set output type - auto output_type = GetImageTypeFromString(output_type_); - if (!output_type) { - return gxf::Unexpected{GXF_FAILURE}; - } - // Check if no-op is needed - no_op_ = output_type.value() == input_info_.type; - return ImageInfo{output_type.value(), input_info_.width, input_info_.height, input_info_.is_cpu}; -} - -template -gxf_result_t ConvertColorFormatBase::doUpdateCameraMessage(gxf::Handle& output, - gxf::Handle& input) { - *output = *input; - return GXF_SUCCESS; -} - -#define DEFINE_CONVERT_COLOR_FORMAT(INPUT_TYPE, OUTPUT_TYPE, CONVERSION_TYPE) \ - if (input_info_.type == INPUT_TYPE && output_info_.type == OUTPUT_TYPE) { \ - return detail::ConvertColorFormatImpl( \ - output, input, output_info_, input_info_, output_name, input_name, output_adapter_.get(), input_adapter_.get(), \ - pool_.get(), CONVERSION_TYPE, stream); \ - } - -#define DEFINE_STREAM_CONVERT_COLOR_FORMAT(INPUT_TYPE, OUTPUT_TYPE) \ - if (input_info_.type == INPUT_TYPE && output_info_.type == OUTPUT_TYPE) { \ - return detail::ConvertColorFormatStreamImpl( \ - output, input, output_info_, input_info_, output_name, input_name, stream_.try_get().value(), \ - output_adapter_.get(), input_adapter_.get(), pool_.get()); \ - } - -template<> -gxf_result_t ConvertColorFormatBase::doExecute(gxf::Entity& output, gxf::Entity& input, cudaStream_t stream, - const char* output_name, const char* input_name) { - GXF_LOG_INFO("execute convert color format"); - - // Run the color conversion operation - DEFINE_STREAM_CONVERT_COLOR_FORMAT(::cvcore::ImageType::RGB_U8, ::cvcore::ImageType::BGR_U8); - DEFINE_STREAM_CONVERT_COLOR_FORMAT(::cvcore::ImageType::BGR_U8, ::cvcore::ImageType::RGB_U8); - DEFINE_STREAM_CONVERT_COLOR_FORMAT(::cvcore::ImageType::NV12, ::cvcore::ImageType::BGR_U8); - DEFINE_STREAM_CONVERT_COLOR_FORMAT(::cvcore::ImageType::BGR_U8, ::cvcore::ImageType::NV12); - DEFINE_STREAM_CONVERT_COLOR_FORMAT(::cvcore::ImageType::NV12, ::cvcore::ImageType::RGB_U8); - DEFINE_STREAM_CONVERT_COLOR_FORMAT(::cvcore::ImageType::RGB_U8, ::cvcore::ImageType::NV12); - DEFINE_STREAM_CONVERT_COLOR_FORMAT(::cvcore::ImageType::NV24, ::cvcore::ImageType::BGR_U8); - DEFINE_STREAM_CONVERT_COLOR_FORMAT(::cvcore::ImageType::BGR_U8, ::cvcore::ImageType::NV24); - DEFINE_STREAM_CONVERT_COLOR_FORMAT(::cvcore::ImageType::NV24, ::cvcore::ImageType::RGB_U8); - DEFINE_STREAM_CONVERT_COLOR_FORMAT(::cvcore::ImageType::RGB_U8, ::cvcore::ImageType::NV24); - - // Return error code for unsupported type - GXF_LOG_ERROR("invalid input/output type for image color conversion."); - return GXF_FAILURE; -} - -template<> -gxf_result_t ConvertColorFormatBase::doExecute(gxf::Entity& output, gxf::Entity& input, cudaStream_t stream, - const char* output_name, const char* input_name) { - GXF_LOG_INFO("execute convert color format"); - - // Run the color conversion operation - DEFINE_CONVERT_COLOR_FORMAT(::cvcore::ImageType::RGB_U8, ::cvcore::ImageType::BGR_U8, - ::cvcore::tensor_ops::ColorConversionType::RGB2BGR); - DEFINE_CONVERT_COLOR_FORMAT(::cvcore::ImageType::RGB_U16, ::cvcore::ImageType::BGR_U16, - ::cvcore::tensor_ops::ColorConversionType::RGB2BGR); - DEFINE_CONVERT_COLOR_FORMAT(::cvcore::ImageType::RGB_F32, ::cvcore::ImageType::BGR_F32, - ::cvcore::tensor_ops::ColorConversionType::RGB2BGR); - DEFINE_CONVERT_COLOR_FORMAT(::cvcore::ImageType::BGR_U8, ::cvcore::ImageType::RGB_U8, - ::cvcore::tensor_ops::ColorConversionType::BGR2RGB); - DEFINE_CONVERT_COLOR_FORMAT(::cvcore::ImageType::BGR_U16, ::cvcore::ImageType::RGB_U16, - ::cvcore::tensor_ops::ColorConversionType::BGR2RGB); - DEFINE_CONVERT_COLOR_FORMAT(::cvcore::ImageType::BGR_F32, ::cvcore::ImageType::RGB_F32, - ::cvcore::tensor_ops::ColorConversionType::BGR2RGB); - DEFINE_CONVERT_COLOR_FORMAT(::cvcore::ImageType::RGB_U8, ::cvcore::ImageType::Y_U8, - ::cvcore::tensor_ops::ColorConversionType::RGB2GRAY); - DEFINE_CONVERT_COLOR_FORMAT(::cvcore::ImageType::RGB_U16, ::cvcore::ImageType::Y_U16, - ::cvcore::tensor_ops::ColorConversionType::RGB2GRAY); - DEFINE_CONVERT_COLOR_FORMAT(::cvcore::ImageType::RGB_F32, ::cvcore::ImageType::Y_F32, - ::cvcore::tensor_ops::ColorConversionType::RGB2GRAY); - DEFINE_CONVERT_COLOR_FORMAT(::cvcore::ImageType::BGR_U8, ::cvcore::ImageType::Y_U8, - ::cvcore::tensor_ops::ColorConversionType::BGR2GRAY); - DEFINE_CONVERT_COLOR_FORMAT(::cvcore::ImageType::BGR_U16, ::cvcore::ImageType::Y_U16, - ::cvcore::tensor_ops::ColorConversionType::BGR2GRAY); - DEFINE_CONVERT_COLOR_FORMAT(::cvcore::ImageType::BGR_F32, ::cvcore::ImageType::Y_F32, - ::cvcore::tensor_ops::ColorConversionType::BGR2GRAY); - DEFINE_CONVERT_COLOR_FORMAT(::cvcore::ImageType::Y_U8, ::cvcore::ImageType::RGB_U8, - ::cvcore::tensor_ops::ColorConversionType::GRAY2RGB); - DEFINE_CONVERT_COLOR_FORMAT(::cvcore::ImageType::Y_U16, ::cvcore::ImageType::RGB_U16, - ::cvcore::tensor_ops::ColorConversionType::GRAY2RGB); - DEFINE_CONVERT_COLOR_FORMAT(::cvcore::ImageType::Y_F32, ::cvcore::ImageType::RGB_F32, - ::cvcore::tensor_ops::ColorConversionType::GRAY2RGB); - DEFINE_CONVERT_COLOR_FORMAT(::cvcore::ImageType::Y_U8, ::cvcore::ImageType::BGR_U8, - ::cvcore::tensor_ops::ColorConversionType::GRAY2BGR); - DEFINE_CONVERT_COLOR_FORMAT(::cvcore::ImageType::Y_U16, ::cvcore::ImageType::BGR_U16, - ::cvcore::tensor_ops::ColorConversionType::GRAY2BGR); - DEFINE_CONVERT_COLOR_FORMAT(::cvcore::ImageType::Y_F32, ::cvcore::ImageType::BGR_F32, - ::cvcore::tensor_ops::ColorConversionType::GRAY2BGR); - - // Return error code for unsupported type - GXF_LOG_ERROR("invalid input/output type for image color conversion."); - return GXF_FAILURE; -} - -template class ConvertColorFormatBase; -template class ConvertColorFormatBase; - -} // namespace tensor_ops -} // namespace cvcore -} // namespace nvidia diff --git a/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/ConvertColorFormat.hpp b/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/ConvertColorFormat.hpp deleted file mode 100644 index f78786b..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/ConvertColorFormat.hpp +++ /dev/null @@ -1,51 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 -#ifndef NVIDIA_CVCORE_CONVERT_COLOR_FORMAT_HPP -#define NVIDIA_CVCORE_CONVERT_COLOR_FORMAT_HPP - -#include "TensorOperator.hpp" - -namespace nvidia { -namespace cvcore { -namespace tensor_ops { - -// ConvertColorFormat operator. -template -class ConvertColorFormatBase : public TensorOperator { -public: - virtual ~ConvertColorFormatBase() {} - - gxf_result_t registerInterface(gxf::Registrar* registrar) override final; - -private: - gxf::Expected doInferOutputInfo(gxf::Entity& input) override final; - gxf_result_t doUpdateCameraMessage(gxf::Handle& output, - gxf::Handle& input) override final; - gxf_result_t doExecute(gxf::Entity& output, gxf::Entity& input, cudaStream_t stream, const char* output_name, - const char* input_name) override final; - - gxf::Parameter output_type_; -}; - -class ConvertColorFormat : public ConvertColorFormatBase {}; -class StreamConvertColorFormat : public ConvertColorFormatBase {}; - -} // namespace tensor_ops -} // namespace cvcore -} // namespace nvidia - -#endif diff --git a/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/CropAndResize.cpp b/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/CropAndResize.cpp deleted file mode 100644 index d0c2872..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/CropAndResize.cpp +++ /dev/null @@ -1,161 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 -#include "CropAndResize.hpp" -#include "Resize.hpp" - -namespace nvidia { -namespace cvcore { -namespace tensor_ops { - -namespace detail { - -template<::cvcore::ImageType T> -gxf_result_t CropAndResizeImpl(gxf::Entity& output, gxf::Entity& input, const ImageInfo& output_info, - const ImageInfo& input_info, const char* output_name, const char* input_name, - gxf::Handle output_adapter, gxf::Handle input_adapter, - gxf::Handle allocator, const std::vector<::cvcore::BBox>& src_rois, - ::cvcore::tensor_ops::InterpolationType interp_type, cudaStream_t stream) { - auto input_image = input_adapter->WrapImageFromMessage(input, input_name); - if (!input_image) { - return GXF_FAILURE; - } - - const size_t num_output = src_rois.size(); - - for (size_t i = 0; i < num_output; i++) { - const std::string output_name_i = std::string(output_name) + "_" + std::to_string(i); - auto error = output_adapter->AddImageToMessage(output, output_info.width, output_info.height, allocator, - output_info.is_cpu, output_name_i.c_str()); - if (error != GXF_SUCCESS) { - return GXF_FAILURE; - } - auto output_image = output_adapter->WrapImageFromMessage(output, output_name_i.c_str()); - if (!output_image) { - return GXF_FAILURE; - } - ::cvcore::tensor_ops::CropAndResize(output_image.value(), input_image.value(), src_rois[i], interp_type, stream); - } - - return GXF_SUCCESS; -} - -} // namespace detail - -gxf_result_t CropAndResize::registerInterface(gxf::Registrar* registrar) { - gxf::Expected result; - - result &= registrar->parameter(output_width_, "output_width"); - result &= registrar->parameter(output_height_, "output_height"); - result &= registrar->parameter(interp_type_, "interp_type"); - result &= registrar->parameter(keep_aspect_ratio_, "keep_aspect_ratio"); - result &= registrar->parameter(receiver_bbox_, "receiver_bbox"); - result &= registrar->parameter(receiver_, "receiver"); - result &= registrar->parameter(transmitter_, "transmitter"); - result &= registrar->parameter(pool_, "pool"); - result &= registrar->parameter(stream_pool_, "stream_pool", "cuda stream pool", "cuda stream pool object", - gxf::Registrar::NoDefaultParameter(), GXF_PARAMETER_FLAGS_OPTIONAL); - result &= registrar->parameter(input_adapter_, "input_adapter"); - result &= registrar->parameter(output_adapter_, "output_adapter"); - result &= registrar->parameter(input_name_, "input_name", "input name", "input tensor name", - gxf::Registrar::NoDefaultParameter(), GXF_PARAMETER_FLAGS_OPTIONAL); - result &= registrar->parameter(output_name_, "output_name", "output name", "output tensor name", - gxf::Registrar::NoDefaultParameter(), GXF_PARAMETER_FLAGS_OPTIONAL); - - return gxf::ToResultCode(result); -} - -gxf::Expected CropAndResize::doInferOutputInfo(gxf::Entity& input) { - // Set crop regions - auto input_bbox_message = receiver_bbox_->receive(); - if (!input_bbox_message) { - return gxf::Unexpected{GXF_FAILURE}; - } - auto bbox_tensor = input_bbox_message.value().get(); - if (!bbox_tensor) { - return gxf::Unexpected{GXF_FAILURE}; - } - const gxf::Shape bbox_shape = bbox_tensor.value()->shape(); - if (bbox_shape.rank() != 2 || bbox_shape.dimension(1) != 4) { - GXF_LOG_ERROR("invalid input bbox dimension."); - return gxf::Unexpected{GXF_FAILURE}; - } - const size_t num_bbox = bbox_shape.dimension(0); - auto bbox_pointer = bbox_tensor.value()->data(); - if (!bbox_pointer) { - GXF_LOG_ERROR("empty bbox input."); - return gxf::Unexpected{GXF_FAILURE}; - } - std::vector<::cvcore::BBox> rois; - for (size_t i = 0; i < num_bbox; i++) { - const int index = i * 4; - rois.push_back({bbox_pointer.value()[index], bbox_pointer.value()[index + 1], bbox_pointer.value()[index + 2], - bbox_pointer.value()[index + 3]}); - } - input_rois_ = std::move(rois); - // Check if no-op is needed - no_op_ = input_rois_.size() == 1 && input_rois_[0].xmin == 0 && - input_rois_[0].xmax == static_cast(input_info_.width) && input_rois_[0].ymin == 0 && - input_rois_[0].ymax == static_cast(input_info_.height); - - return ImageInfo{input_info_.type, output_width_.get(), output_height_.get(), input_info_.is_cpu}; -} - -gxf_result_t CropAndResize::doUpdateCameraMessage(gxf::Handle& output, - gxf::Handle& input) { - auto crop_result = GetCroppedCameraModel(*input, input_rois_[0]); - if (!crop_result) { - return GXF_FAILURE; - } - *output = GetScaledCameraModel(crop_result.value(), output_info_.width, output_info_.height, false).value(); - return GXF_SUCCESS; -} - -#define DEFINE_CROP_AND_RESIZE(INPUT_TYPE) \ - if (input_info_.type == INPUT_TYPE) { \ - return detail::CropAndResizeImpl(output, input, output_info_, input_info_, output_name, input_name, \ - output_adapter_.get(), input_adapter_.get(), pool_.get(), \ - input_rois_, interp.value(), stream); \ - } - -gxf_result_t CropAndResize::doExecute(gxf::Entity& output, gxf::Entity& input, cudaStream_t stream, - const char* output_name, const char* input_name) { - GXF_LOG_INFO("execute crop_and_resize."); - // Check if interpolation type is valid - auto interp = GetInterpolationType(interp_type_); - if (!interp) { - return interp.error(); - } - - // Run the image resizing operation - DEFINE_CROP_AND_RESIZE(::cvcore::ImageType::Y_U8); - DEFINE_CROP_AND_RESIZE(::cvcore::ImageType::Y_U16); - DEFINE_CROP_AND_RESIZE(::cvcore::ImageType::Y_F32); - DEFINE_CROP_AND_RESIZE(::cvcore::ImageType::RGB_U8); - DEFINE_CROP_AND_RESIZE(::cvcore::ImageType::RGB_U16); - DEFINE_CROP_AND_RESIZE(::cvcore::ImageType::RGB_F32); - DEFINE_CROP_AND_RESIZE(::cvcore::ImageType::BGR_U8); - DEFINE_CROP_AND_RESIZE(::cvcore::ImageType::BGR_U16); - DEFINE_CROP_AND_RESIZE(::cvcore::ImageType::BGR_F32); - - // Return error code for unsupported type - GXF_LOG_ERROR("invalid input/output type for image crop_and_resize."); - return GXF_FAILURE; -} - -} // namespace tensor_ops -} // namespace cvcore -} // namespace nvidia diff --git a/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/CropAndResize.hpp b/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/CropAndResize.hpp deleted file mode 100644 index 2bde6fb..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/CropAndResize.hpp +++ /dev/null @@ -1,53 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 -#ifndef NVIDIA_CVCORE_CROP_AND_RESIZE_HPP -#define NVIDIA_CVCORE_CROP_AND_RESIZE_HPP - -#include "TensorOperator.hpp" -#include "cv/core/BBox.h" - -namespace nvidia { -namespace cvcore { -namespace tensor_ops { - -// CropAndResize operator. -class CropAndResize : public TensorOperator { -public: - virtual ~CropAndResize() {} - - gxf_result_t registerInterface(gxf::Registrar* registrar) override; - -private: - gxf::Expected doInferOutputInfo(gxf::Entity& input) override final; - gxf_result_t doUpdateCameraMessage(gxf::Handle& output, - gxf::Handle& input) override final; - gxf_result_t doExecute(gxf::Entity& output, gxf::Entity& input, cudaStream_t stream, const char* output_name, - const char* input_name) override final; - - gxf::Parameter output_width_; - gxf::Parameter output_height_; - gxf::Parameter interp_type_; - gxf::Parameter keep_aspect_ratio_; - gxf::Parameter> receiver_bbox_; - std::vector<::cvcore::BBox> input_rois_; -}; - -} // namespace tensor_ops -} // namespace cvcore -} // namespace nvidia - -#endif diff --git a/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/Frame3D.cpp b/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/Frame3D.cpp deleted file mode 100644 index c28829f..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/Frame3D.cpp +++ /dev/null @@ -1,56 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -#include "Frame3D.hpp" - -namespace nvidia { -namespace cvcore { -namespace tensor_ops { - -gxf_result_t Frame3D::registerInterface(gxf::Registrar* registrar) { - gxf::Expected result; - - result &= registrar->parameter(rotation_, "rotation"); - result &= registrar->parameter(translation_, "translation"); - - return gxf::ToResultCode(result); -} - -gxf_result_t Frame3D::initialize() { - // Construct extrinsic model - if (rotation_.get().size() != 9) { - GXF_LOG_ERROR("size of rotation matrix must be 9"); - return GXF_FAILURE; - } - if (translation_.get().size() != 3) { - GXF_LOG_ERROR("size of translation vector must be 3"); - return GXF_FAILURE; - } - float raw_matrix[3][4]; - for (size_t i = 0; i < 9; i++) { - raw_matrix[i / 3][i % 3] = rotation_.get()[i]; - } - for (size_t i = 0; i < 3; i++) { - raw_matrix[i][3] = translation_.get()[i]; - } - extrinsics_ = ::cvcore::CameraExtrinsics(raw_matrix); - return GXF_SUCCESS; -} - -} // namespace tensor_ops -} // namespace cvcore -} // namespace nvidia diff --git a/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/Frame3D.hpp b/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/Frame3D.hpp deleted file mode 100644 index aec8cf5..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/Frame3D.hpp +++ /dev/null @@ -1,53 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 -#ifndef NVIDIA_CVCORE_FRAME3D_HPP -#define NVIDIA_CVCORE_FRAME3D_HPP - -#include "gxf/core/component.hpp" -#include "gxf/std/parameter_parser_std.hpp" - -#include "cv/core/CameraModel.h" - -namespace nvidia { -namespace cvcore { -namespace tensor_ops { - -// Wrapper of CameraExtrinsics compatible with CVCORE -class Frame3D : public gxf::Component { -public: - virtual ~Frame3D() = default; - Frame3D() = default; - - gxf_result_t registerInterface(gxf::Registrar* registrar) override; - gxf_result_t initialize() override; - - ::cvcore::CameraExtrinsics getCameraExtrinsics() const { - return extrinsics_; - } - -private: - gxf::Parameter> rotation_; - gxf::Parameter> translation_; - - ::cvcore::CameraExtrinsics extrinsics_; -}; - -} // namespace tensor_ops -} // namespace cvcore -} // namespace nvidia - -#endif diff --git a/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/ImageAdapter.cpp b/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/ImageAdapter.cpp deleted file mode 100644 index 12e7fe7..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/ImageAdapter.cpp +++ /dev/null @@ -1,78 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 -#include "ImageAdapter.hpp" - -namespace nvidia { -namespace cvcore { -namespace tensor_ops { - -gxf_result_t ImageAdapter::registerInterface(gxf::Registrar* registrar) { - gxf::Expected result; - - result &= registrar->parameter(message_type_param_, "message_type"); - result &= registrar->parameter(image_type_param_, "image_type", "image type", "optional image type", - gxf::Registrar::NoDefaultParameter(), GXF_PARAMETER_FLAGS_OPTIONAL); - result &= registrar->parameter(allocate_pitch_linear_, "allocate_pitch_linear", - "if true, allocate output buffers as padded pitch linear surfaces", "", false); - - return gxf::ToResultCode(result); -} - -gxf_result_t ImageAdapter::initialize() { - if (message_type_param_.get() == "Tensor") { - message_type_ = BufferType::TENSOR; - } else if (message_type_param_.get() == "VideoBuffer") { - message_type_ = BufferType::VIDEO_BUFFER; - } else { - GXF_LOG_ERROR("unknown buffer type."); - return GXF_FAILURE; - } - - const auto& image_type_param = image_type_param_.try_get(); - if (message_type_ == BufferType::TENSOR && !image_type_param) { - GXF_LOG_INFO("image type must be specified for gxf::Tensor."); - return GXF_FAILURE; - } - if (image_type_param) { - const auto image_type = GetImageTypeFromString(image_type_param.value()); - if (!image_type) { - return GXF_FAILURE; - } - image_type_ = image_type.value(); - } - return GXF_SUCCESS; -} - -gxf::Expected ImageAdapter::GetImageInfo(const gxf::Entity& message, const char* name) { - if (message_type_ == BufferType::TENSOR) { - auto tensor = message.get(name); - if (!tensor) { - return gxf::Unexpected{GXF_FAILURE}; - } - return detail::GetTensorInfo(tensor.value(), image_type_); - } else { - auto video_buffer = message.get(name); - if (!video_buffer) { - return gxf::Unexpected{GXF_FAILURE}; - } - return detail::GetVideoBufferInfo(video_buffer.value()); - } -} - -} // namespace tensor_ops -} // namespace cvcore -} // namespace nvidia diff --git a/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/ImageAdapter.hpp b/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/ImageAdapter.hpp deleted file mode 100644 index 3b41391..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/ImageAdapter.hpp +++ /dev/null @@ -1,101 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 -#ifndef NVIDIA_CVCORE_IMAGE_ADAPTER_HPP -#define NVIDIA_CVCORE_IMAGE_ADAPTER_HPP - -#include "ImageUtils.hpp" -#include "detail/ImageAdapterTensorImpl.hpp" -#include "detail/ImageAdapterVideoBufferImpl.hpp" - -#include "gxf/core/component.hpp" -#include "gxf/multimedia/video.hpp" -#include "gxf/std/allocator.hpp" -#include "gxf/std/parameter_parser_std.hpp" -#include "gxf/std/tensor.hpp" - -#include "cv/core/Image.h" - -namespace nvidia { -namespace cvcore { -namespace tensor_ops { - -// Enum class: gxf::Tensor and gxf::VideoBuffer -enum class BufferType { - TENSOR, - VIDEO_BUFFER, -}; - -// Utility component for conversion between message and cvcore image type -class ImageAdapter : public gxf::Component { -public: - virtual ~ImageAdapter() = default; - ImageAdapter() = default; - - gxf_result_t registerInterface(gxf::Registrar* registrar) override; - gxf_result_t initialize() override; - - gxf::Expected GetImageInfo(const gxf::Entity& message, const char* name = nullptr); - - template<::cvcore::ImageType T> - gxf::Expected<::cvcore::Image> WrapImageFromMessage(const gxf::Entity& message, const char* name = nullptr) { - if (message_type_ == BufferType::TENSOR) { - auto tensor = message.get(name); - if (!tensor) { - return gxf::Unexpected{GXF_FAILURE}; - } - return detail::WrapImageFromTensor(tensor.value()); - } else { - auto video_buffer = message.get(name); - if (!video_buffer) { - return gxf::Unexpected{GXF_FAILURE}; - } - return detail::WrapImageFromVideoBuffer(video_buffer.value()); - } - } - - template<::cvcore::ImageType T> - gxf_result_t AddImageToMessage(gxf::Entity& message, size_t width, size_t height, - gxf::Handle allocator, bool is_cpu, const char* name = nullptr) { - if (message_type_ == BufferType::TENSOR) { - auto tensor = message.add(name); - if (!tensor) { - return GXF_FAILURE; - } - return detail::AllocateTensor(tensor.value(), width, height, allocator, is_cpu, allocate_pitch_linear_.get()); - } else { - auto video_buffer = message.add(name); - if (!video_buffer) { - return GXF_FAILURE; - } - return detail::AllocateVideoBuffer(video_buffer.value(), width, height, allocator, is_cpu, allocate_pitch_linear_.get()); - } - } - -private: - gxf::Parameter message_type_param_; - gxf::Parameter image_type_param_; - gxf::Parameter allocate_pitch_linear_; - - ::cvcore::ImageType image_type_; - BufferType message_type_; -}; - -} // namespace tensor_ops -} // namespace cvcore -} // namespace nvidia - -#endif diff --git a/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/ImageUtils.cpp b/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/ImageUtils.cpp deleted file mode 100644 index ffb5bcc..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/ImageUtils.cpp +++ /dev/null @@ -1,175 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 -#include "ImageUtils.hpp" - -#include -#include - -namespace nvidia { -namespace cvcore { -namespace tensor_ops { - -// helper function to match input image type string to cvcore::ImageType -gxf::Expected<::cvcore::ImageType> GetImageTypeFromString(const std::string& type) { - if (type == "Y_U8") { - return ::cvcore::ImageType::Y_U8; - } else if (type == "Y_U16") { - return ::cvcore::ImageType::Y_U16; - } else if (type == "Y_F32") { - return ::cvcore::ImageType::Y_F32; - } else if (type == "RGB_U8") { - return ::cvcore::ImageType::RGB_U8; - } else if (type == "RGB_U16") { - return ::cvcore::ImageType::RGB_U16; - } else if (type == "RGB_F32") { - return ::cvcore::ImageType::RGB_F32; - } else if (type == "BGR_U8") { - return ::cvcore::ImageType::BGR_U8; - } else if (type == "BGR_U16") { - return ::cvcore::ImageType::BGR_U16; - } else if (type == "BGR_F32") { - return ::cvcore::ImageType::BGR_F32; - } else if (type == "PLANAR_RGB_U8") { - return ::cvcore::ImageType::PLANAR_RGB_U8; - } else if (type == "PLANAR_RGB_U16") { - return ::cvcore::ImageType::PLANAR_RGB_U16; - } else if (type == "PLANAR_RGB_F32") { - return ::cvcore::ImageType::PLANAR_RGB_F32; - } else if (type == "PLANAR_BGR_U8") { - return ::cvcore::ImageType::PLANAR_BGR_U8; - } else if (type == "PLANAR_BGR_U16") { - return ::cvcore::ImageType::PLANAR_BGR_U16; - } else if (type == "PLANAR_BGR_F32") { - return ::cvcore::ImageType::PLANAR_BGR_F32; - } else if (type == "NV12") { - return ::cvcore::ImageType::NV12; - } else if (type == "NV24") { - return ::cvcore::ImageType::NV24; - } else { - GXF_LOG_ERROR("invalid image type."); - return gxf::Unexpected{GXF_FAILURE}; - } -} - -gxf::Expected<::cvcore::tensor_ops::InterpolationType> GetInterpolationType(const std::string& type) { - if (type == "nearest") { - return ::cvcore::tensor_ops::InterpolationType::INTERP_NEAREST; - } else if (type == "linear") { - return ::cvcore::tensor_ops::InterpolationType::INTERP_LINEAR; - } else if (type == "cubic_bspline") { - return ::cvcore::tensor_ops::InterpolationType::INTERP_CUBIC_BSPLINE; - } else if (type == "cubic_catmullrom") { - return ::cvcore::tensor_ops::InterpolationType::INTERP_CUBIC_CATMULLROM; - } else { - GXF_LOG_ERROR("invalid interpolation type."); - return gxf::Unexpected{GXF_FAILURE}; - } -} - -gxf::Expected<::cvcore::tensor_ops::BorderType> GetBorderType(const std::string& type) { - if (type == "zero") { - return ::cvcore::tensor_ops::BorderType::BORDER_ZERO; - } else if (type == "repeat") { - return ::cvcore::tensor_ops::BorderType::BORDER_REPEAT; - } else if (type == "reverse") { - return ::cvcore::tensor_ops::BorderType::BORDER_REVERSE; - } else if (type == "mirror") { - return ::cvcore::tensor_ops::BorderType::BORDER_MIRROR; - } else { - GXF_LOG_ERROR("invalid border type."); - return gxf::Unexpected{GXF_FAILURE}; - } -} - -gxf::Expected<::cvcore::CameraDistortionType> GetCameraDistortionType(gxf::DistortionType type) { - switch (type) { - case gxf::DistortionType::Perspective: - return ::cvcore::CameraDistortionType::NONE; - case gxf::DistortionType::Polynomial: - return ::cvcore::CameraDistortionType::Polynomial; - case gxf::DistortionType::FisheyeEquidistant: - return ::cvcore::CameraDistortionType::FisheyeEquidistant; - case gxf::DistortionType::FisheyeEquisolid: - return ::cvcore::CameraDistortionType::FisheyeEquisolid; - case gxf::DistortionType::FisheyeOrthoGraphic: - return ::cvcore::CameraDistortionType::FisheyeOrthoGraphic; - case gxf::DistortionType::FisheyeStereographic: - return ::cvcore::CameraDistortionType::FisheyeStereographic; - default: - GXF_LOG_ERROR("invalid distortion type."); - return gxf::Unexpected{GXF_FAILURE}; - } -} - -gxf::Expected GetDistortionType(::cvcore::CameraDistortionType type) { - switch (type) { - case ::cvcore::CameraDistortionType::Polynomial: - return gxf::DistortionType::Polynomial; - case ::cvcore::CameraDistortionType::FisheyeEquidistant: - return gxf::DistortionType::FisheyeEquidistant; - case ::cvcore::CameraDistortionType::FisheyeEquisolid: - return gxf::DistortionType::FisheyeEquisolid; - case ::cvcore::CameraDistortionType::FisheyeOrthoGraphic: - return gxf::DistortionType::FisheyeOrthoGraphic; - case ::cvcore::CameraDistortionType::FisheyeStereographic: - return gxf::DistortionType::FisheyeStereographic; - default: - GXF_LOG_ERROR("invalid distortion type."); - return gxf::Unexpected{GXF_FAILURE}; - } -} - -gxf::Expected GetCroppedCameraModel(const gxf::CameraModel& input, const ::cvcore::BBox& roi) { - if (!roi.isValid()) { - return gxf::Unexpected{GXF_FAILURE}; - } - gxf::CameraModel camera; - const size_t output_width = roi.xmax - roi.xmin; - const size_t output_height = roi.ymax - roi.ymin; - camera.dimensions = {static_cast(output_width), static_cast(output_height)}; - camera.focal_length = input.focal_length; - // We will keep the relative principal point location unchanged for cropping; - camera.principal_point = {input.principal_point.x / input.dimensions.x * output_width, - input.principal_point.y / input.dimensions.y * output_height}, - camera.skew_value = input.skew_value; - camera.distortion_type = input.distortion_type; - std::copy(std::begin(input.distortion_coefficients), std::end(input.distortion_coefficients), - std::begin(camera.distortion_coefficients)); - return camera; -} - -gxf::Expected GetScaledCameraModel(const gxf::CameraModel& input, size_t output_width, - size_t output_height, bool keep_aspect_ratio) { - gxf::CameraModel camera; - const float scaler_x = static_cast(output_width) / input.dimensions.x; - const float scaler_y = static_cast(output_height) / input.dimensions.y; - const float min_scaler = std::min(scaler_x, scaler_y); - camera.dimensions = {static_cast(output_width), static_cast(output_height)}; - camera.focal_length = keep_aspect_ratio - ? nvidia::gxf::Vector2f{min_scaler * input.focal_length.x, min_scaler * input.focal_length.y} - : nvidia::gxf::Vector2f{scaler_x * input.focal_length.x, scaler_y * input.focal_length.y}; - camera.principal_point = {scaler_x * input.principal_point.x, scaler_y * input.principal_point.y}, - camera.skew_value = input.skew_value; - camera.distortion_type = input.distortion_type; - std::copy(std::begin(input.distortion_coefficients), std::end(input.distortion_coefficients), - std::begin(camera.distortion_coefficients)); - return camera; -} - -} // namespace tensor_ops -} // namespace cvcore -} // namespace nvidia diff --git a/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/ImageUtils.hpp b/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/ImageUtils.hpp deleted file mode 100644 index d052827..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/ImageUtils.hpp +++ /dev/null @@ -1,65 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 -#ifndef NVIDIA_CVCORE_IMAGE_UTILS_HPP -#define NVIDIA_CVCORE_IMAGE_UTILS_HPP - -#include "cv/core/BBox.h" -#include "cv/core/CameraModel.h" -#include "cv/core/Image.h" -#include "cv/tensor_ops/ImageUtils.h" -#include "gxf/core/expected.hpp" -#include "gxf/multimedia/camera.hpp" - -namespace nvidia { -namespace cvcore { -namespace tensor_ops { - -// Description of Image -struct ImageInfo { - ::cvcore::ImageType type; - size_t width; - size_t height; - bool is_cpu; -}; - -// helper function to match input image type string to cvcore::ImageType -gxf::Expected<::cvcore::ImageType> GetImageTypeFromString(const std::string& type); - -// Helper function to get the interpolation type -gxf::Expected<::cvcore::tensor_ops::InterpolationType> GetInterpolationType(const std::string& type); - -// Helper function to get the border type -gxf::Expected<::cvcore::tensor_ops::BorderType> GetBorderType(const std::string& type); - -// Helper function to get the cvcore camera distortion type -gxf::Expected<::cvcore::CameraDistortionType> GetCameraDistortionType(gxf::DistortionType type); - -// Helper function to get the gxf distortion type -gxf::Expected GetDistortionType(::cvcore::CameraDistortionType type); - -// Helper function to get the new camera model after applying crop operation -gxf::Expected GetCroppedCameraModel(const gxf::CameraModel& input, const ::cvcore::BBox& roi); - -// Helper function to get the new camera model after applying scale operation -gxf::Expected GetScaledCameraModel(const gxf::CameraModel& input, size_t output_width, - size_t output_height, bool keep_aspect_ratio); - -} // namespace tensor_ops -} // namespace cvcore -} // namespace nvidia - -#endif diff --git a/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/InterleavedToPlanar.cpp b/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/InterleavedToPlanar.cpp deleted file mode 100644 index 7b63825..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/InterleavedToPlanar.cpp +++ /dev/null @@ -1,146 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 -#include "InterleavedToPlanar.hpp" - -namespace nvidia { -namespace cvcore { -namespace tensor_ops { - -namespace detail { - -template<::cvcore::ImageType T_IN, ::cvcore::ImageType T_OUT> -gxf_result_t InterleavedToPlanarImpl(gxf::Entity& output, gxf::Entity& input, const ImageInfo& output_info, - const ImageInfo& input_info, const char* output_name, const char* input_name, - gxf::Handle output_adapter, gxf::Handle input_adapter, - gxf::Handle allocator, cudaStream_t stream) { - auto input_image = input_adapter->WrapImageFromMessage(input, input_name); - if (!input_image) { - return GXF_FAILURE; - } - - auto error = output_adapter->AddImageToMessage(output, output_info.width, output_info.height, allocator, - output_info.is_cpu, output_name); - if (error != GXF_SUCCESS) { - return GXF_FAILURE; - } - - auto output_image = output_adapter->WrapImageFromMessage(output, output_name); - if (!output_image) { - return GXF_FAILURE; - } - ::cvcore::tensor_ops::InterleavedToPlanar(output_image.value(), input_image.value(), stream); - return GXF_SUCCESS; -} - -} // namespace detail - -gxf_result_t InterleavedToPlanar::registerInterface(gxf::Registrar* registrar) { - gxf::Expected result; - - result &= registrar->parameter(receiver_, "receiver"); - result &= registrar->parameter(transmitter_, "transmitter"); - result &= registrar->parameter(pool_, "pool"); - result &= registrar->parameter(stream_pool_, "stream_pool", "cuda stream pool", "cuda stream pool object", - gxf::Registrar::NoDefaultParameter(), GXF_PARAMETER_FLAGS_OPTIONAL); - result &= registrar->parameter(input_adapter_, "input_adapter"); - result &= registrar->parameter(output_adapter_, "output_adapter"); - result &= registrar->parameter(input_name_, "input_name", "input name", "input tensor name", - gxf::Registrar::NoDefaultParameter(), GXF_PARAMETER_FLAGS_OPTIONAL); - result &= registrar->parameter(output_name_, "output_name", "output name", "output tensor name", - gxf::Registrar::NoDefaultParameter(), GXF_PARAMETER_FLAGS_OPTIONAL); - - return gxf::ToResultCode(result); -} - -gxf::Expected InterleavedToPlanar::doInferOutputInfo(gxf::Entity& input) { - // Output type is planar - ::cvcore::ImageType output_type; - switch (input_info_.type) { - case ::cvcore::ImageType::RGB_U8: { - output_type = ::cvcore::ImageType::PLANAR_RGB_U8; - break; - } - case ::cvcore::ImageType::RGB_U16: { - output_type = ::cvcore::ImageType::PLANAR_RGB_U16; - break; - } - case ::cvcore::ImageType::RGB_F32: { - output_type = ::cvcore::ImageType::PLANAR_RGB_F32; - break; - } - case ::cvcore::ImageType::BGR_U8: { - output_type = ::cvcore::ImageType::PLANAR_BGR_U8; - break; - } - case ::cvcore::ImageType::BGR_U16: { - output_type = ::cvcore::ImageType::PLANAR_BGR_U16; - break; - } - case ::cvcore::ImageType::BGR_F32: { - output_type = ::cvcore::ImageType::PLANAR_BGR_F32; - break; - } - case ::cvcore::ImageType::PLANAR_RGB_U8: - case ::cvcore::ImageType::PLANAR_RGB_U16: - case ::cvcore::ImageType::PLANAR_RGB_F32: - case ::cvcore::ImageType::PLANAR_BGR_U8: - case ::cvcore::ImageType::PLANAR_BGR_U16: - case ::cvcore::ImageType::PLANAR_BGR_F32: { - output_type = input_info_.type; - no_op_ = true; - break; - } - default: { - GXF_LOG_ERROR("invalid input type for interleaved to planar conversion."); - return gxf::Unexpected{GXF_FAILURE}; - } - } - return ImageInfo{output_type, input_info_.width, input_info_.height, input_info_.is_cpu}; -} - -gxf_result_t InterleavedToPlanar::doUpdateCameraMessage(gxf::Handle& output, - gxf::Handle& input) { - *output = *input; - return GXF_SUCCESS; -} - -#define DEFINE_INTERLEAVED_TO_PLANAR(INPUT_TYPE, OUTPUT_TYPE) \ - if (input_info_.type == INPUT_TYPE && output_info_.type == OUTPUT_TYPE) { \ - return detail::InterleavedToPlanarImpl(output, input, output_info_, input_info_, \ - output_name, input_name, output_adapter_.get(), \ - input_adapter_.get(), pool_.get(), stream); \ - } - -gxf_result_t InterleavedToPlanar::doExecute(gxf::Entity& output, gxf::Entity& input, cudaStream_t stream, - const char* output_name, const char* input_name) { - GXF_LOG_INFO("execute interleaved_to_planar conversion"); - // Run the interleaved to planar operation - DEFINE_INTERLEAVED_TO_PLANAR(::cvcore::ImageType::RGB_U8, ::cvcore::ImageType::PLANAR_RGB_U8); - DEFINE_INTERLEAVED_TO_PLANAR(::cvcore::ImageType::RGB_U16, ::cvcore::ImageType::PLANAR_RGB_U16); - DEFINE_INTERLEAVED_TO_PLANAR(::cvcore::ImageType::RGB_F32, ::cvcore::ImageType::PLANAR_RGB_F32); - DEFINE_INTERLEAVED_TO_PLANAR(::cvcore::ImageType::BGR_U8, ::cvcore::ImageType::PLANAR_BGR_U8); - DEFINE_INTERLEAVED_TO_PLANAR(::cvcore::ImageType::BGR_U16, ::cvcore::ImageType::PLANAR_BGR_U16); - DEFINE_INTERLEAVED_TO_PLANAR(::cvcore::ImageType::BGR_F32, ::cvcore::ImageType::PLANAR_BGR_F32); - - // Return error code for unsupported type - GXF_LOG_ERROR("invalid input/output type for image interleaved to planar conversion."); - return GXF_FAILURE; -} - -} // namespace tensor_ops -} // namespace cvcore -} // namespace nvidia diff --git a/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/InterleavedToPlanar.hpp b/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/InterleavedToPlanar.hpp deleted file mode 100644 index 2e1efbe..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/InterleavedToPlanar.hpp +++ /dev/null @@ -1,45 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 -#ifndef NVIDIA_CVCORE_INTERLEAVED_TO_PLANAR_HPP -#define NVIDIA_CVCORE_INTERLEAVED_TO_PLANAR_HPP - -#include "TensorOperator.hpp" - -namespace nvidia { -namespace cvcore { -namespace tensor_ops { - -// InterleavedToPlanar operator. -class InterleavedToPlanar : public TensorOperator { -public: - virtual ~InterleavedToPlanar() {} - - gxf_result_t registerInterface(gxf::Registrar* registrar) override; - -private: - gxf::Expected doInferOutputInfo(gxf::Entity& input) override final; - gxf_result_t doUpdateCameraMessage(gxf::Handle& output, - gxf::Handle& input) override final; - gxf_result_t doExecute(gxf::Entity& output, gxf::Entity& input, cudaStream_t stream, const char* output_name, - const char* input_name) override final; -}; - -} // namespace tensor_ops -} // namespace cvcore -} // namespace nvidia - -#endif diff --git a/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/Normalize.cpp b/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/Normalize.cpp deleted file mode 100644 index 2429abb..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/Normalize.cpp +++ /dev/null @@ -1,183 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 -#include "Normalize.hpp" - -namespace nvidia { -namespace cvcore { -namespace tensor_ops { - -namespace detail { - -template<::cvcore::ImageType T_IN, ::cvcore::ImageType T_OUT> -gxf_result_t NormalizeC1Impl(gxf::Entity& output, gxf::Entity& input, const ImageInfo& output_info, - const ImageInfo& input_info, const char* output_name, const char* input_name, - gxf::Handle output_adapter, gxf::Handle input_adapter, - gxf::Handle allocator, const std::vector& scales, - const std::vector& offsets, cudaStream_t stream) { - if (scales.size() != 1 || offsets.size() != 1) { - GXF_LOG_ERROR("invalid scales/offsets dimension"); - return GXF_FAILURE; - } - - auto input_image = input_adapter->WrapImageFromMessage(input, input_name); - if (!input_image) { - return GXF_FAILURE; - } - - auto error = output_adapter->AddImageToMessage(output, output_info.width, output_info.height, allocator, - output_info.is_cpu, output_name); - if (error != GXF_SUCCESS) { - return GXF_FAILURE; - } - - auto output_image = output_adapter->WrapImageFromMessage(output, output_name); - if (!output_image) { - return GXF_FAILURE; - } - ::cvcore::tensor_ops::Normalize(output_image.value(), input_image.value(), scales[0], offsets[0], stream); - return GXF_SUCCESS; -} - -template<::cvcore::ImageType T_IN, ::cvcore::ImageType T_OUT> -gxf_result_t NormalizeC3Impl(gxf::Entity& output, gxf::Entity& input, const ImageInfo& output_info, - const ImageInfo& input_info, const char* output_name, const char* input_name, - gxf::Handle output_adapter, gxf::Handle input_adapter, - gxf::Handle allocator, const std::vector& scales, - const std::vector& offsets, cudaStream_t stream) { - if (scales.size() != 3 || offsets.size() != 3) { - GXF_LOG_ERROR("invalid scales/offsets dimension"); - return GXF_FAILURE; - } - - auto input_image = input_adapter->WrapImageFromMessage(input, input_name); - if (!input_image) { - return GXF_FAILURE; - } - - auto error = output_adapter->AddImageToMessage(output, output_info.width, output_info.height, allocator, - output_info.is_cpu, output_name); - if (error != GXF_SUCCESS) { - return GXF_FAILURE; - } - - auto output_image = output_adapter->WrapImageFromMessage(output, output_name); - if (!output_image) { - return GXF_FAILURE; - } - const float scales_value[3] = {scales[0], scales[1], scales[2]}; - const float offsets_value[3] = {offsets[0], offsets[1], offsets[2]}; - ::cvcore::tensor_ops::Normalize(output_image.value(), input_image.value(), scales_value, offsets_value, stream); - return GXF_SUCCESS; -} - -} // namespace detail - -gxf_result_t Normalize::registerInterface(gxf::Registrar* registrar) { - gxf::Expected result; - - result &= registrar->parameter(scales_, "scales"); - result &= registrar->parameter(offsets_, "offsets"); - result &= registrar->parameter(receiver_, "receiver"); - result &= registrar->parameter(transmitter_, "transmitter"); - result &= registrar->parameter(pool_, "pool"); - result &= registrar->parameter(stream_pool_, "stream_pool", "cuda stream pool", "cuda stream pool object", - gxf::Registrar::NoDefaultParameter(), GXF_PARAMETER_FLAGS_OPTIONAL); - result &= registrar->parameter(input_adapter_, "input_adapter"); - result &= registrar->parameter(output_adapter_, "output_adapter"); - result &= registrar->parameter(input_name_, "input_name", "input name", "input tensor name", - gxf::Registrar::NoDefaultParameter(), GXF_PARAMETER_FLAGS_OPTIONAL); - result &= registrar->parameter(output_name_, "output_name", "output name", "output tensor name", - gxf::Registrar::NoDefaultParameter(), GXF_PARAMETER_FLAGS_OPTIONAL); - - return gxf::ToResultCode(result); -} - -gxf::Expected Normalize::doInferOutputInfo(gxf::Entity& input) { - // Output type is F32 - ::cvcore::ImageType output_type; - switch (input_info_.type) { - case ::cvcore::ImageType::Y_U8: - case ::cvcore::ImageType::Y_U16: - case ::cvcore::ImageType::Y_F32: { - output_type = ::cvcore::ImageType::Y_F32; - break; - } - case ::cvcore::ImageType::RGB_U8: - case ::cvcore::ImageType::RGB_U16: - case ::cvcore::ImageType::RGB_F32: { - output_type = ::cvcore::ImageType::RGB_F32; - break; - } - case ::cvcore::ImageType::BGR_U8: - case ::cvcore::ImageType::BGR_U16: - case ::cvcore::ImageType::BGR_F32: { - output_type = ::cvcore::ImageType::BGR_F32; - break; - } - default: { - GXF_LOG_ERROR("invalid input type for normalize."); - return gxf::Unexpected{GXF_FAILURE}; - } - } - // Operation must be performed under any condition - no_op_ = false; - return ImageInfo{output_type, input_info_.width, input_info_.height, input_info_.is_cpu}; -} - -gxf_result_t Normalize::doUpdateCameraMessage(gxf::Handle& output, - gxf::Handle& input) { - *output = *input; - return GXF_SUCCESS; -} - -#define DEFINE_NORMALIZE_C1(INPUT_TYPE, OUTPUT_TYPE) \ - if (input_info_.type == INPUT_TYPE && output_info_.type == OUTPUT_TYPE) { \ - return detail::NormalizeC1Impl(output, input, output_info_, input_info_, output_name, \ - input_name, output_adapter_.get(), input_adapter_.get(), \ - pool_.get(), scales_.get(), offsets_.get(), stream); \ - } - -#define DEFINE_NORMALIZE_C3(INPUT_TYPE, OUTPUT_TYPE) \ - if (input_info_.type == INPUT_TYPE && output_info_.type == OUTPUT_TYPE) { \ - return detail::NormalizeC3Impl(output, input, output_info_, input_info_, output_name, \ - input_name, output_adapter_.get(), input_adapter_.get(), \ - pool_.get(), scales_.get(), offsets_.get(), stream); \ - } - -gxf_result_t Normalize::doExecute(gxf::Entity& output, gxf::Entity& input, cudaStream_t stream, const char* output_name, - const char* input_name) { - GXF_LOG_INFO("execute normalize"); - - // Run the image normalization operation - DEFINE_NORMALIZE_C1(::cvcore::ImageType::Y_U8, ::cvcore::ImageType::Y_F32); - DEFINE_NORMALIZE_C1(::cvcore::ImageType::Y_U16, ::cvcore::ImageType::Y_F32); - DEFINE_NORMALIZE_C1(::cvcore::ImageType::Y_F32, ::cvcore::ImageType::Y_F32); - DEFINE_NORMALIZE_C3(::cvcore::ImageType::RGB_U8, ::cvcore::ImageType::RGB_F32); - DEFINE_NORMALIZE_C3(::cvcore::ImageType::RGB_U16, ::cvcore::ImageType::RGB_F32); - DEFINE_NORMALIZE_C3(::cvcore::ImageType::RGB_F32, ::cvcore::ImageType::RGB_F32); - DEFINE_NORMALIZE_C3(::cvcore::ImageType::BGR_U8, ::cvcore::ImageType::BGR_F32); - DEFINE_NORMALIZE_C3(::cvcore::ImageType::BGR_U16, ::cvcore::ImageType::BGR_F32); - DEFINE_NORMALIZE_C3(::cvcore::ImageType::BGR_F32, ::cvcore::ImageType::BGR_F32); - - // Return error code for unsupported type - GXF_LOG_ERROR("invalid input/output type for image normalize."); - return GXF_FAILURE; -} - -} // namespace tensor_ops -} // namespace cvcore -} // namespace nvidia diff --git a/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/Normalize.hpp b/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/Normalize.hpp deleted file mode 100644 index efb735a..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/Normalize.hpp +++ /dev/null @@ -1,47 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 -#ifndef NVIDIA_CVCORE_NORMALIZE_HPP -#define NVIDIA_CVCORE_NORMALIZE_HPP - -#include "TensorOperator.hpp" - -namespace nvidia { -namespace cvcore { -namespace tensor_ops { - -// Normalization operator. -class Normalize : public TensorOperator { -public: - virtual ~Normalize() {} - gxf_result_t registerInterface(gxf::Registrar* registrar) override; - -private: - gxf::Expected doInferOutputInfo(gxf::Entity& input) override final; - gxf_result_t doUpdateCameraMessage(gxf::Handle& output, - gxf::Handle& input) override final; - gxf_result_t doExecute(gxf::Entity& output, gxf::Entity& input, cudaStream_t stream, const char* output_name, - const char* input_name) override final; - - gxf::Parameter> scales_; - gxf::Parameter> offsets_; -}; - -} // namespace tensor_ops -} // namespace cvcore -} // namespace nvidia - -#endif diff --git a/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/Reshape.cpp b/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/Reshape.cpp deleted file mode 100644 index 322b843..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/Reshape.cpp +++ /dev/null @@ -1,98 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 -#include "Reshape.hpp" - -namespace nvidia { -namespace cvcore { -namespace tensor_ops { - -gxf_result_t Reshape::registerInterface(gxf::Registrar* registrar) { - gxf::Expected result; - - result &= registrar->parameter(output_shape_, "output_shape"); - result &= registrar->parameter(receiver_, "receiver"); - result &= registrar->parameter(transmitter_, "transmitter"); - result &= registrar->parameter(pool_, "pool"); - result &= registrar->parameter(input_adapter_, "input_adapter"); - result &= registrar->parameter(output_adapter_, "output_adapter"); - result &= registrar->parameter(input_name_, "input_name", "input name", "input tensor name", - gxf::Registrar::NoDefaultParameter(), GXF_PARAMETER_FLAGS_OPTIONAL); - result &= registrar->parameter(output_name_, "output_name", "output name", "output tensor name", - gxf::Registrar::NoDefaultParameter(), GXF_PARAMETER_FLAGS_OPTIONAL); - - return gxf::ToResultCode(result); -} - -gxf_result_t Reshape::doUpdateCameraMessage(gxf::Handle& output, - gxf::Handle& input) { - *output = *input; - return GXF_SUCCESS; -} - -gxf_result_t Reshape::doExecute(gxf::Entity& output, gxf::Entity& input, cudaStream_t stream, const char* output_name, - const char* input_name) { - GXF_LOG_INFO("execute reshape."); - - auto input_tensor = input.get(input_name); - if (!input_tensor) { - GXF_LOG_ERROR("input message does not contain Tensor"); - return input_tensor.error(); - } - - auto output_tensor = output.add(output_name); - if (!output_tensor) { - GXF_LOG_ERROR("unable to add output Tensor"); - return output_tensor.error(); - } - - const auto& input_shape = input_tensor.value()->shape(); - const std::vector& output_shape_arr = output_shape_; - std::array dims = {}; - std::copy(output_shape_arr.begin(), output_shape_arr.end(), dims.begin()); - const auto output_shape = gxf::Shape(dims, output_shape_arr.size()); - - if (output_shape.size() != input_shape.size()) { - GXF_LOG_ERROR("reshape size mismatch."); - return GXF_FAILURE; - } - - auto result = output_tensor.value()->reshapeCustom( - output_shape, input_tensor.value()->element_type(), gxf::PrimitiveTypeSize(input_tensor.value()->element_type()), - gxf::Unexpected{GXF_UNINITIALIZED_VALUE}, input_tensor.value()->storage_type(), pool_.get()); - - if (!result) { - GXF_LOG_ERROR("reshape tensor failed."); - return result.error(); - } - - // Simply copy the memory - if (input_tensor.value()->storage_type() == gxf::MemoryStorageType::kDevice) { - cudaError_t error = cudaMemcpyAsync(output_tensor.value()->pointer(), input_tensor.value()->pointer(), - input_tensor.value()->size(), cudaMemcpyDeviceToDevice, stream); - if (error != cudaSuccess) { - GXF_LOG_ERROR("cudaMemcpyAsync returned error code"); - return GXF_FAILURE; - } - } else { - memcpy(output_tensor.value()->pointer(), input_tensor.value()->pointer(), input_tensor.value()->size()); - } - return GXF_SUCCESS; -} - -} // namespace tensor_ops -} // namespace cvcore -} // namespace nvidia diff --git a/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/Reshape.hpp b/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/Reshape.hpp deleted file mode 100644 index c6f1c6f..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/Reshape.hpp +++ /dev/null @@ -1,49 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 -#ifndef NVIDIA_CVCORE_RESHAPE_HPP -#define NVIDIA_CVCORE_RESHAPE_HPP - -#include "TensorOperator.hpp" - -namespace nvidia { -namespace cvcore { -namespace tensor_ops { - -// Reshaping operator (only valid for gxf::Tensor). -class Reshape : public TensorOperator { -public: - virtual ~Reshape() {} - gxf_result_t registerInterface(gxf::Registrar* registrar) override; - -private: - gxf::Expected doInferOutputInfo(gxf::Entity& input) override final { - no_op_ = false; - return ImageInfo{}; - }; - gxf_result_t doUpdateCameraMessage(gxf::Handle& output, - gxf::Handle& input) override final; - gxf_result_t doExecute(gxf::Entity& output, gxf::Entity& input, cudaStream_t stream, const char* output_name, - const char* input_name) override final; - - gxf::Parameter> output_shape_; -}; - -} // namespace tensor_ops -} // namespace cvcore -} // namespace nvidia - -#endif diff --git a/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/Resize.cpp b/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/Resize.cpp deleted file mode 100644 index 943ac5a..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/Resize.cpp +++ /dev/null @@ -1,194 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 -#include "Resize.hpp" - -namespace nvidia { -namespace cvcore { -namespace tensor_ops { - -namespace detail { - -template<::cvcore::ImageType T> -gxf_result_t ResizeImpl(gxf::Entity& output, gxf::Entity& input, const ImageInfo& output_info, - const ImageInfo& input_info, const char* output_name, const char* input_name, - gxf::Handle output_adapter, gxf::Handle input_adapter, - gxf::Handle allocator, bool keep_aspect_ratio, - ::cvcore::tensor_ops::InterpolationType interp_type, cudaStream_t stream) { - auto input_image = input_adapter->WrapImageFromMessage(input, input_name); - if (!input_image) { - return GXF_FAILURE; - } - - auto error = output_adapter->AddImageToMessage(output, output_info.width, output_info.height, allocator, - output_info.is_cpu, output_name); - if (error != GXF_SUCCESS) { - return GXF_FAILURE; - } - - auto output_image = output_adapter->WrapImageFromMessage(output, output_name); - if (!output_image) { - return GXF_FAILURE; - } - ::cvcore::tensor_ops::Resize(output_image.value(), input_image.value(), keep_aspect_ratio, interp_type, stream); - return GXF_SUCCESS; -} - -template<::cvcore::ImageType T> -gxf_result_t ResizeStreamImpl(gxf::Entity& output, gxf::Entity& input, const ImageInfo& output_info, - const ImageInfo& input_info, const char* output_name, const char* input_name, - gxf::Handle stream, gxf::Handle output_adapter, - gxf::Handle input_adapter, gxf::Handle allocator, - ::cvcore::tensor_ops::InterpolationType interp_type, - ::cvcore::tensor_ops::BorderType border_type) { - auto input_image = input_adapter->WrapImageFromMessage(input, input_name); - if (!input_image) { - return GXF_FAILURE; - } - - auto error = output_adapter->AddImageToMessage(output, output_info.width, output_info.height, allocator, - output_info.is_cpu, output_name); - if (error != GXF_SUCCESS) { - return GXF_FAILURE; - } - - auto output_image = output_adapter->WrapImageFromMessage(output, output_name); - if (!output_image) { - return GXF_FAILURE; - } - - auto err_code = stream->getStream()->Resize(output_image.value(), input_image.value(), interp_type, border_type); - if (err_code != ::cvcore::make_error_condition(::cvcore::ErrorCode::SUCCESS)) { - GXF_LOG_ERROR("resize operation failed."); - return GXF_FAILURE; - } - - return GXF_SUCCESS; -} - -} // namespace detail - -template -gxf_result_t ResizeBase::registerInterface(gxf::Registrar* registrar) { - gxf::Expected result; - - result &= registrar->parameter(output_width_, "output_width"); - result &= registrar->parameter(output_height_, "output_height"); - result &= registrar->parameter(interp_type_, "interp_type"); - result &= registrar->parameter(border_type_, "border_type"); - result &= registrar->parameter(keep_aspect_ratio_, "keep_aspect_ratio"); - result &= registrar->parameter(receiver_, "receiver"); - result &= registrar->parameter(transmitter_, "transmitter"); - result &= registrar->parameter(pool_, "pool"); - result &= registrar->parameter(stream_, "stream", "tensor stream", "tensor stream object", - gxf::Registrar::NoDefaultParameter(), GXF_PARAMETER_FLAGS_OPTIONAL); - result &= registrar->parameter(stream_pool_, "stream_pool", "cuda stream pool", "cuda stream pool object", - gxf::Registrar::NoDefaultParameter(), GXF_PARAMETER_FLAGS_OPTIONAL); - result &= registrar->parameter(input_adapter_, "input_adapter"); - result &= registrar->parameter(output_adapter_, "output_adapter"); - result &= registrar->parameter(input_name_, "input_name", "input name", "input tensor name", - gxf::Registrar::NoDefaultParameter(), GXF_PARAMETER_FLAGS_OPTIONAL); - result &= registrar->parameter(output_name_, "output_name", "output name", "output tensor name", - gxf::Registrar::NoDefaultParameter(), GXF_PARAMETER_FLAGS_OPTIONAL); - - return gxf::ToResultCode(result); -} - -template -gxf::Expected ResizeBase::doInferOutputInfo(gxf::Entity& input) { - // Check if no-op is needed - no_op_ = output_width_.get() == input_info_.width && output_height_.get() == input_info_.height; - return ImageInfo{input_info_.type, output_width_.get(), output_height_.get(), input_info_.is_cpu}; -} - -template -gxf_result_t ResizeBase::doUpdateCameraMessage(gxf::Handle& output, - gxf::Handle& input) { - *output = GetScaledCameraModel(*input, output_info_.width, output_info_.height, keep_aspect_ratio_.get()).value(); - return GXF_SUCCESS; -} - -#define DEFINE_RESIZE(INPUT_TYPE) \ - if (input_info_.type == INPUT_TYPE) { \ - return detail::ResizeImpl(output, input, output_info_, input_info_, output_name, input_name, \ - output_adapter_.get(), input_adapter_.get(), pool_.get(), \ - keep_aspect_ratio_.get(), interp.value(), stream); \ - } - -#define DEFINE_STREAM_RESIZE(INPUT_TYPE) \ - if (input_info_.type == INPUT_TYPE) { \ - return detail::ResizeStreamImpl(output, input, output_info_, input_info_, output_name, input_name, \ - stream_.try_get().value(), output_adapter_.get(), \ - input_adapter_.get(), pool_.get(), interp.value(), border.value()); \ - } - -template<> -gxf_result_t ResizeBase::doExecute(gxf::Entity& output, gxf::Entity& input, cudaStream_t stream, - const char* output_name, const char* input_name) { - GXF_LOG_INFO("execute resize."); - // Check if interpolation type is valid - auto interp = GetInterpolationType(interp_type_); - if (!interp) { - return interp.error(); - } - auto border = GetBorderType(border_type_); - if (!border) { - return border.error(); - } - - // Run the image resizing operation - DEFINE_STREAM_RESIZE(::cvcore::ImageType::RGB_U8); - DEFINE_STREAM_RESIZE(::cvcore::ImageType::BGR_U8); - DEFINE_STREAM_RESIZE(::cvcore::ImageType::NV12); - DEFINE_STREAM_RESIZE(::cvcore::ImageType::NV24); - - // Return error code for unsupported type - GXF_LOG_ERROR("invalid input/output type for image resize."); - return GXF_FAILURE; -} - -template<> -gxf_result_t ResizeBase::doExecute(gxf::Entity& output, gxf::Entity& input, cudaStream_t stream, - const char* output_name, const char* input_name) { - GXF_LOG_INFO("execute resize."); - // Check if interpolation type is valid - auto interp = GetInterpolationType(interp_type_); - if (!interp) { - return interp.error(); - } - - // Run the image resizing operation - DEFINE_RESIZE(::cvcore::ImageType::Y_U8); - DEFINE_RESIZE(::cvcore::ImageType::Y_U16); - DEFINE_RESIZE(::cvcore::ImageType::Y_F32); - DEFINE_RESIZE(::cvcore::ImageType::RGB_U8); - DEFINE_RESIZE(::cvcore::ImageType::RGB_U16); - DEFINE_RESIZE(::cvcore::ImageType::RGB_F32); - DEFINE_RESIZE(::cvcore::ImageType::BGR_U8); - DEFINE_RESIZE(::cvcore::ImageType::BGR_U16); - DEFINE_RESIZE(::cvcore::ImageType::BGR_F32); - - // Return error code for unsupported type - GXF_LOG_ERROR("invalid input/output type for image resize."); - return GXF_FAILURE; -} - -template class ResizeBase; -template class ResizeBase; - -} // namespace tensor_ops -} // namespace cvcore -} // namespace nvidia diff --git a/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/Resize.hpp b/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/Resize.hpp deleted file mode 100644 index 6771e6e..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/Resize.hpp +++ /dev/null @@ -1,55 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 -#ifndef NVIDIA_CVCORE_RESIZE_HPP -#define NVIDIA_CVCORE_RESIZE_HPP - -#include "TensorOperator.hpp" - -namespace nvidia { -namespace cvcore { -namespace tensor_ops { - -// Resizing operator. -template -class ResizeBase : public TensorOperator { -public: - virtual ~ResizeBase() {} - - gxf_result_t registerInterface(gxf::Registrar* registrar) override; - -private: - gxf::Expected doInferOutputInfo(gxf::Entity& input) override final; - gxf_result_t doUpdateCameraMessage(gxf::Handle& output, - gxf::Handle& input) override final; - gxf_result_t doExecute(gxf::Entity& output, gxf::Entity& input, cudaStream_t stream, const char* output_name, - const char* input_name) override final; - - gxf::Parameter output_width_; - gxf::Parameter output_height_; - gxf::Parameter interp_type_; - gxf::Parameter border_type_; - gxf::Parameter keep_aspect_ratio_; -}; - -class Resize : public ResizeBase {}; -class StreamResize : public ResizeBase {}; - -} // namespace tensor_ops -} // namespace cvcore -} // namespace nvidia - -#endif diff --git a/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/TensorOperator.cpp b/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/TensorOperator.cpp deleted file mode 100644 index 294fb06..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/TensorOperator.cpp +++ /dev/null @@ -1,235 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2020-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 -#include "TensorOperator.hpp" - -#include "gxf/std/timestamp.hpp" - -namespace nvidia { -namespace cvcore { -namespace tensor_ops { - -namespace detail { - -// Function to bind a cuda stream with cid into downstream message -gxf_result_t BindCudaStream(gxf::Entity& message, gxf_uid_t cid) { - if (cid == kNullUid) { - GXF_LOG_ERROR("stream_cid is null"); - return GXF_FAILURE; - } - auto output_stream_id = message.add("stream"); - if (!output_stream_id) { - GXF_LOG_ERROR("failed to add cudastreamid."); - return GXF_FAILURE; - } - output_stream_id.value()->stream_cid = cid; - return GXF_SUCCESS; -} - -// Function to record a new cuda event -gxf_result_t RecordCudaEvent(gxf::Entity& message, gxf::Handle& stream) { - // Create a new event - cudaEvent_t cuda_event; - cudaEventCreateWithFlags(&cuda_event, 0); - gxf::CudaEvent event; - auto ret = event.initWithEvent(cuda_event, stream->dev_id(), [](auto) {}); - if (!ret) { - GXF_LOG_ERROR("failed to init cuda event"); - return GXF_FAILURE; - } - // Record the event - // Can define []() { GXF_LOG_DEBUG("tensorops event synced"); } as callback func for debug purpose - ret = stream->record(event.event().value(), - [event = cuda_event, entity = message.clone().value()](auto) { cudaEventDestroy(event); }); - if (!ret) { - GXF_LOG_ERROR("record event failed"); - return ret.error(); - } - return GXF_SUCCESS; -} - -template -gxf_result_t RerouteMessage(gxf::Entity& output, gxf::Entity& input, - std::function, gxf::Handle)> func, - const char* name = nullptr) { - auto maybe_component = input.get(); - if (maybe_component) { - auto output_component = output.add(name != nullptr ? name : maybe_component.value().name()); - if (!output_component) { - GXF_LOG_ERROR("add output component failed."); - return output_component.error(); - } - return func(output_component.value(), maybe_component.value()); - } - return GXF_SUCCESS; -} - -} // namespace detail - -gxf_result_t TensorOperator::inferOutputInfo(gxf::Entity& input) { - const char* input_name = input_name_.try_get() ? input_name_.try_get().value().c_str() : nullptr; - auto input_info = input_adapter_.get()->GetImageInfo(input, input_name); - if (!input_info) { - return input_info.error(); - } - input_info_ = input_info.value(); - auto output_info = doInferOutputInfo(input); - if (!output_info) { - return output_info.error(); - } - output_info_ = output_info.value(); - return GXF_SUCCESS; -} - -gxf_result_t TensorOperator::updateCameraMessage(gxf::Handle& output, - gxf::Handle& input) { - return doUpdateCameraMessage(output, input); -} - -gxf_result_t TensorOperator::execute(gxf::Entity& output, gxf::Entity& input, cudaStream_t stream) { - const char* output_name = output_name_.try_get() ? output_name_.try_get().value().c_str() : nullptr; - const char* input_name = input_name_.try_get() ? input_name_.try_get().value().c_str() : nullptr; - return doExecute(output, input, stream, output_name, input_name); -} - -gxf_result_t TensorOperator::start() { - // Allocate cuda stream using stream pool if necessary - if (stream_pool_.try_get()) { - auto stream = stream_pool_.try_get().value()->allocateStream(); - if (!stream) { - GXF_LOG_ERROR("allocating stream failed."); - return GXF_FAILURE; - } - cuda_stream_ptr_ = std::move(stream.value()); - if (!cuda_stream_ptr_->stream()) { - GXF_LOG_ERROR("allocated stream is not initialized."); - return GXF_FAILURE; - } - } - return GXF_SUCCESS; -} - -gxf_result_t TensorOperator::tick() { - // Receiving the data - auto input_message = receiver_->receive(); - // Check received message for errors - if (!input_message) { - return input_message.error(); - } - // Infer output ImageInfo and if it's no-op - auto error = inferOutputInfo(input_message.value()); - if (error != GXF_SUCCESS) { - return error; - } - // Re-direct the input message if no-op is needed - if (no_op_) { - transmitter_->publish(input_message.value()); - return GXF_SUCCESS; - } - // Create output message - gxf::Expected output_message = gxf::Entity::New(context()); - if (!output_message) { - return output_message.error(); - } - // Pass through timestamp if presented in input message - error = - detail::RerouteMessage(output_message.value(), input_message.value(), - [](gxf::Handle output, gxf::Handle input) { - *output = *input; - return GXF_SUCCESS; - }); - if (error != GXF_SUCCESS) { - return error; - } - // Pass through cudaStreamId or create a new cuda stream for NPP backend only - cudaStream_t cuda_stream = 0; // default stream - if (!stream_.try_get()) { - // Allocate new CudaStream if StreamPool attached - if (stream_pool_.try_get()) { - cuda_stream = cuda_stream_ptr_->stream().value(); - if (detail::BindCudaStream(output_message.value(), cuda_stream_ptr_.cid()) != GXF_SUCCESS) { - return GXF_FAILURE; - } - } - auto input_stream_id = input_message.value().get(); - if (input_stream_id) { - auto stream = - gxf::Handle::Create(input_stream_id.value().context(), input_stream_id.value()->stream_cid); - if (!stream) { - GXF_LOG_ERROR("create cudastream from cid failed."); - return GXF_FAILURE; - } - if (stream_pool_.try_get()) { - // sync upstreaming input cuda stream - if (!stream.value()->syncStream()) { - GXF_LOG_ERROR("sync stream failed."); - return GXF_FAILURE; - } - } else { - cuda_stream = stream.value()->stream().value(); - if (detail::BindCudaStream(output_message.value(), stream.value().cid()) != GXF_SUCCESS) { - return GXF_FAILURE; - } - cuda_stream_ptr_ = stream.value(); - } - } - } - // Execute the operation - error = execute(output_message.value(), input_message.value(), cuda_stream); - if (error != GXF_SUCCESS) { - GXF_LOG_ERROR("operation failed."); - return GXF_FAILURE; - } - // Record the cuda event if necessary - if (!cuda_stream_ptr_.is_null()) { - // record on both input/output stream - if (detail::RecordCudaEvent(input_message.value(), cuda_stream_ptr_) != GXF_SUCCESS) { - return GXF_FAILURE; - } - if (detail::RecordCudaEvent(output_message.value(), cuda_stream_ptr_) != GXF_SUCCESS) { - return GXF_FAILURE; - } - } - // Update output camera message if necessary - error = detail::RerouteMessage( - output_message.value(), input_message.value(), - [this](gxf::Handle output, gxf::Handle input) { - return updateCameraMessage(output, input); - }, - "camera"); - if (error != GXF_SUCCESS) { - return error; - } - // Pass through pose3d message if necessary - error = detail::RerouteMessage( - output_message.value(), input_message.value(), - [](gxf::Handle output, gxf::Handle input) { - *output = *input; - return GXF_SUCCESS; - }, - "pose"); - if (error != GXF_SUCCESS) { - return error; - } - // Send the processed data - transmitter_->publish(output_message.value()); - - return GXF_SUCCESS; -} - -} // namespace tensor_ops -} // namespace cvcore -} // namespace nvidia diff --git a/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/TensorOperator.hpp b/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/TensorOperator.hpp deleted file mode 100644 index 4c65d47..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/TensorOperator.hpp +++ /dev/null @@ -1,95 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 -#ifndef NVIDIA_CVCORE_TENSOR_OPERATOR_HPP -#define NVIDIA_CVCORE_TENSOR_OPERATOR_HPP - -#include "ImageAdapter.hpp" -#include "ImageUtils.hpp" -#include "TensorStream.hpp" - -#include "gxf/cuda/cuda_stream.hpp" -#include "gxf/cuda/cuda_stream_id.hpp" -#include "gxf/cuda/cuda_stream_pool.hpp" -#include "gxf/std/allocator.hpp" -#include "gxf/std/codelet.hpp" -#include "gxf/std/parameter_parser_std.hpp" -#include "gxf/std/receiver.hpp" -#include "gxf/std/tensor.hpp" -#include "gxf/std/transmitter.hpp" - -#include "cv/core/Image.h" -#include "cv/core/Tensor.h" -#include "cv/tensor_ops/ImageUtils.h" - -namespace nvidia { -namespace cvcore { -namespace tensor_ops { - -// Base class for all tensor_ops operators -class TensorOperator : public gxf::Codelet { -public: - virtual ~TensorOperator() = default; - - gxf_result_t inferOutputInfo(gxf::Entity& input); - - gxf_result_t updateCameraMessage(gxf::Handle& output, gxf::Handle& input); - - gxf_result_t execute(gxf::Entity& output, gxf::Entity& input, cudaStream_t stream); - - gxf_result_t start() override; - - gxf_result_t tick() override; - - virtual gxf_result_t stop() override { - return GXF_SUCCESS; - } - -protected: - gxf::Parameter> receiver_; - gxf::Parameter> transmitter_; - gxf::Parameter> pool_; - gxf::Parameter> stream_; - gxf::Parameter> stream_pool_; - gxf::Parameter> input_adapter_; - gxf::Parameter> output_adapter_; - gxf::Parameter input_name_; - gxf::Parameter output_name_; - - // Input image info - ImageInfo input_info_; - // Output image info - ImageInfo output_info_; - // Whether to skip the operation(by default is false) - bool no_op_ = false; - -private: - virtual gxf::Expected doInferOutputInfo(gxf::Entity& input) = 0; - - virtual gxf_result_t doUpdateCameraMessage(gxf::Handle& output, - gxf::Handle& input) = 0; - - virtual gxf_result_t doExecute(gxf::Entity& output, gxf::Entity& input, cudaStream_t stream, const char* output_name, - const char* input_name) = 0; - - gxf::Handle cuda_stream_ptr_ = nullptr; -}; - -} // namespace tensor_ops -} // namespace cvcore -} // namespace nvidia - -#endif diff --git a/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/TensorOps.cpp b/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/TensorOps.cpp deleted file mode 100644 index 8b62548..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/TensorOps.cpp +++ /dev/null @@ -1,91 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 -#include "CameraModel.hpp" -#include "ConvertColorFormat.hpp" -#include "CropAndResize.hpp" -#include "Frame3D.hpp" -#include "ImageAdapter.hpp" -#include "InterleavedToPlanar.hpp" -#include "Normalize.hpp" -#include "Reshape.hpp" -#include "Resize.hpp" -#include "TensorOperator.hpp" -#include "TensorStream.hpp" -#include "Undistort.hpp" - -#include "gxf/std/extension_factory_helper.hpp" - -GXF_EXT_FACTORY_BEGIN() -GXF_EXT_FACTORY_SET_INFO( - 0x6eae64ff97a94d9b, 0xb324f85e6a98a75a, "NvCvTensorOpsExtension", - "Generic CVCORE tensor_ops interfaces", "Nvidia_Gxf", "3.1.0", "LICENSE"); - -GXF_EXT_FACTORY_ADD( - 0xd073a92344ba4b82, 0xbd0f18f4996048e8, nvidia::cvcore::tensor_ops::CameraModel, - nvidia::gxf::Component, - "Construct Camera distortion model / Camera intrinsic compatible with CVCORE"); - -GXF_EXT_FACTORY_ADD( - 0x6c9419223e4b4c2c, 0x899a4d65279c6508, nvidia::cvcore::tensor_ops::Frame3D, - nvidia::gxf::Component, - "Construct Camera extrinsic compatible with CVCORE"); - -GXF_EXT_FACTORY_ADD( - 0xd94385e5b35b4635, 0x9adb0d214a3865f7, nvidia::cvcore::tensor_ops::TensorStream, - nvidia::gxf::Component, "Wrapper of CVCORE ITensorOperatorStream/ITensorOperatorContext"); - -GXF_EXT_FACTORY_ADD( - 0xd0c4ddad486a4a92, 0xb69c8a5304b205ea, nvidia::cvcore::tensor_ops::ImageAdapter, - nvidia::gxf::Component, "Utility component for conversion between message and cvcore image type"); - -GXF_EXT_FACTORY_ADD( - 0xadebc792bd0b4a57, 0x99c1405fd2ea0728, nvidia::cvcore::tensor_ops::StreamUndistort, - nvidia::gxf::Codelet, "Codelet for stream image undistortion in tensor_ops"); - -GXF_EXT_FACTORY_ADD( - 0xa58141ac7eca4ea6, 0x9b545446fe379a12, nvidia::cvcore::tensor_ops::Resize, nvidia::gxf::Codelet, - "Codelet for image resizing in tensor_ops"); - -GXF_EXT_FACTORY_ADD( - 0xeb8b5f5b36d44b49, 0x81f959fd28e6f678, nvidia::cvcore::tensor_ops::StreamResize, - nvidia::gxf::Codelet, "Codelet for stream image resizing in tensor_ops"); - -GXF_EXT_FACTORY_ADD( - 0x4a7ff422de3841bd, 0x9e743ac10d9294b7, nvidia::cvcore::tensor_ops::CropAndResize, - nvidia::gxf::Codelet, "Codelet for crop and resizing operation in tensor_ops"); - -GXF_EXT_FACTORY_ADD( - 0x7018f0b9034c462c, 0xa9fbaf7ee012974a, nvidia::cvcore::tensor_ops::Normalize, - nvidia::gxf::Codelet, - "Codelet for image normalization in tensor_ops"); - -GXF_EXT_FACTORY_ADD( - 0x269d4237f3c3479e, 0xbcca9ecc44c71a71, nvidia::cvcore::tensor_ops::InterleavedToPlanar, - nvidia::gxf::Codelet, "Codelet for convert interleaved image to planar image in tensor_ops"); - -GXF_EXT_FACTORY_ADD( - 0xfc4d7b4d8fcc4dab, 0xa286056e0fcafa79, nvidia::cvcore::tensor_ops::ConvertColorFormat, - nvidia::gxf::Codelet, "Codelet for image color conversion in tensor_ops"); - -GXF_EXT_FACTORY_ADD( - 0x5ab4a4d8f7a34553, 0xa90be52660b076fe, nvidia::cvcore::tensor_ops::StreamConvertColorFormat, - nvidia::gxf::Codelet, "Codelet for stream image color conversion in tensor_ops"); - -GXF_EXT_FACTORY_ADD( - 0x26789b7d5a8d4e85, 0x86b845ec5f4cd12b, nvidia::cvcore::tensor_ops::Reshape, nvidia::gxf::Codelet, - "Codelet for image reshape in tensor_ops"); -GXF_EXT_FACTORY_END() diff --git a/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/TensorStream.cpp b/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/TensorStream.cpp deleted file mode 100644 index 2f825cf..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/TensorStream.cpp +++ /dev/null @@ -1,124 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -#include "TensorStream.hpp" - -#include "cv/core/ComputeEngine.h" -#include "cv/tensor_ops/TensorOperators.h" - -namespace nvidia { -namespace cvcore { -namespace tensor_ops { - -namespace detail { - -gxf::Expected<::cvcore::tensor_ops::TensorBackend> GetContextType(const std::string& type) { - if (type == "NPP") { - return ::cvcore::tensor_ops::TensorBackend::NPP; - } else if (type == "VPI") { - return ::cvcore::tensor_ops::TensorBackend::VPI; - } else if (type == "DALI") { - return ::cvcore::tensor_ops::TensorBackend::DALI; - } else { - return gxf::Unexpected{GXF_FAILURE}; - } -} - -gxf::Expected<::cvcore::ComputeEngine> GetComputeEngineType(const std::string& type) { - if (type == "UNKNOWN") { - return ::cvcore::ComputeEngine::UNKNOWN; - } else if (type == "CPU") { - return ::cvcore::ComputeEngine::CPU; - } else if (type == "PVA") { - return ::cvcore::ComputeEngine::PVA; - } else if (type == "VIC") { - return ::cvcore::ComputeEngine::VIC; - } else if (type == "NVENC") { - return ::cvcore::ComputeEngine::NVENC; - } else if (type == "GPU") { - return ::cvcore::ComputeEngine::GPU; - } else if (type == "DLA") { - return ::cvcore::ComputeEngine::DLA; - } else if (type == "COMPUTE_FAULT") { - return ::cvcore::ComputeEngine::COMPUTE_FAULT; - } else { - return gxf::Unexpected{GXF_FAILURE}; - } -} - -} // namespace detail - -gxf_result_t TensorStream::registerInterface(gxf::Registrar* registrar) { - gxf::Expected result; - - result &= registrar->parameter(backend_type_, "backend_type"); - result &= registrar->parameter(engine_type_, "engine_type"); - - return gxf::ToResultCode(result); -} - -gxf_result_t TensorStream::initialize() { - // Construct context - auto backend_type = detail::GetContextType(backend_type_.get()); - if (!backend_type) { - GXF_LOG_ERROR("unknown backend type."); - return GXF_FAILURE; - } - if (!::cvcore::tensor_ops::TensorContextFactory::IsBackendSupported(backend_type.value())) { - GXF_LOG_ERROR("unsupported context type."); - return GXF_FAILURE; - } - auto err_code = ::cvcore::tensor_ops::TensorContextFactory::CreateContext(context_, backend_type.value()); - if (err_code != ::cvcore::make_error_code(::cvcore::ErrorCode::SUCCESS)) { - GXF_LOG_ERROR("tensor context creation failed."); - return GXF_FAILURE; - } - // Construct stream - auto engine_type = detail::GetComputeEngineType(engine_type_.get()); - if (!engine_type) { - return GXF_FAILURE; - } - - if (!context_->IsComputeEngineCompatible(engine_type.value())) { - GXF_LOG_ERROR("invalid compute engine type."); - return GXF_FAILURE; - } - err_code = context_->CreateStream(stream_, engine_type.value()); - if (err_code != ::cvcore::make_error_code(::cvcore::ErrorCode::SUCCESS)) { - GXF_LOG_ERROR("tensor stream creation failed."); - return GXF_FAILURE; - } - return GXF_SUCCESS; -} - -gxf_result_t TensorStream::deinitialize() { - auto err_code = context_->DestroyStream(stream_); - if (err_code != ::cvcore::make_error_code(::cvcore::ErrorCode::SUCCESS)) { - GXF_LOG_ERROR("tensor stream destroy failed."); - return GXF_FAILURE; - } - err_code = ::cvcore::tensor_ops::TensorContextFactory::DestroyContext(context_); - if (err_code != ::cvcore::make_error_code(::cvcore::ErrorCode::SUCCESS)) { - GXF_LOG_ERROR("tensor context destroy failed."); - return GXF_FAILURE; - } - return GXF_SUCCESS; -} - -} // namespace tensor_ops -} // namespace cvcore -} // namespace nvidia diff --git a/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/TensorStream.hpp b/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/TensorStream.hpp deleted file mode 100644 index 710e11f..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/TensorStream.hpp +++ /dev/null @@ -1,59 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 -#ifndef NVIDIA_CVCORE_TENSOR_STREAM_HPP -#define NVIDIA_CVCORE_TENSOR_STREAM_HPP - -#include "gxf/core/component.hpp" -#include "gxf/std/parameter_parser_std.hpp" - -#include "cv/tensor_ops/ITensorOperatorContext.h" -#include "cv/tensor_ops/ITensorOperatorStream.h" - -namespace nvidia { -namespace cvcore { -namespace tensor_ops { - -// Wrapper of CVCORE ITensorOperatorStream/ITensorOperatorContext -class TensorStream : public gxf::Component { -public: - virtual ~TensorStream() = default; - TensorStream() = default; - - gxf_result_t registerInterface(gxf::Registrar* registrar) override; - gxf_result_t initialize() override; - gxf_result_t deinitialize() override; - - ::cvcore::tensor_ops::TensorOperatorContext getContext() const { - return context_; - } - ::cvcore::tensor_ops::TensorOperatorStream getStream() const { - return stream_; - } - -private: - gxf::Parameter backend_type_; - gxf::Parameter engine_type_; - - ::cvcore::tensor_ops::TensorOperatorContext context_; - ::cvcore::tensor_ops::TensorOperatorStream stream_; -}; - -} // namespace tensor_ops -} // namespace cvcore -} // namespace nvidia - -#endif diff --git a/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/Undistort.cpp b/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/Undistort.cpp deleted file mode 100644 index b1eed4e..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/Undistort.cpp +++ /dev/null @@ -1,285 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -#include - -#include "ImageUtils.hpp" -#include "Undistort.hpp" -#include "gxf/multimedia/camera.hpp" - -namespace nvidia { -namespace cvcore { -namespace tensor_ops { - -namespace detail { - -template<::cvcore::ImageType T> -gxf_result_t UndistortImpl(gxf::Entity& output, gxf::Entity& input, const ImageInfo& output_info, - const ImageInfo& input_info, const char* output_name, const char* input_name, - gxf::Handle stream, gxf::Handle output_adapter, - gxf::Handle input_adapter, gxf::Handle allocator, - ::cvcore::tensor_ops::ImageWarp warp, ::cvcore::tensor_ops::InterpolationType interp_type, - ::cvcore::tensor_ops::BorderType border_type) { - auto input_image = input_adapter->WrapImageFromMessage(input, input_name); - if (!input_image) { - return GXF_FAILURE; - } - - auto error = output_adapter->AddImageToMessage(output, output_info.width, output_info.height, allocator, - output_info.is_cpu, output_name); - if (error != GXF_SUCCESS) { - return GXF_FAILURE; - } - - auto output_image = output_adapter->WrapImageFromMessage(output, output_name); - if (!output_image) { - return GXF_FAILURE; - } - - auto err_code = stream->getStream()->Remap(output_image.value(), input_image.value(), warp, interp_type, border_type); - if (err_code != ::cvcore::make_error_condition(::cvcore::ErrorCode::SUCCESS)) { - GXF_LOG_ERROR("undistort operation failed."); - return GXF_FAILURE; - } - - return GXF_SUCCESS; -} - -gxf::Expected<::cvcore::CameraIntrinsics> GetIntrinsicsFromMessage(gxf::Handle& camera_model) { - return ::cvcore::CameraIntrinsics(camera_model->focal_length.x, camera_model->focal_length.y, - camera_model->principal_point.x, camera_model->principal_point.y, - camera_model->skew_value); -} - -gxf::Expected<::cvcore::CameraExtrinsics> GetExtrinsicsFromMessage(gxf::Handle& pose) { - float raw_matrix[3][4]; - for (size_t i = 0; i < 9; i++) { - raw_matrix[i / 3][i % 3] = pose->rotation[i]; - } - for (size_t i = 0; i < 3; i++) { - raw_matrix[i][3] = pose->translation[i]; - } - return ::cvcore::CameraExtrinsics(raw_matrix); -} - -gxf::Expected<::cvcore::CameraDistortionModel> GetDistortionsFromMessage(gxf::Handle& camera_model) { - auto distortion_type = GetCameraDistortionType(camera_model->distortion_type); - if (!distortion_type) { - return gxf::Unexpected{GXF_FAILURE}; - } - auto distortion_model = ::cvcore::CameraDistortionModel(); - for (size_t i = 0; i < 8; i++) { - distortion_model.coefficients[i] = camera_model->distortion_coefficients[i]; - } - distortion_model.type = distortion_type.value(); - return distortion_model; -} - -} // namespace detail - -gxf_result_t UndistortBase::start() { - // Load grid object - image_grid_.numHorizRegions = regions_width_.get().size(); - image_grid_.numVertRegions = regions_height_.get().size(); - if (regions_width_.get().size() != horizontal_intervals_.get().size() || - regions_height_.get().size() != vertical_intervals_.get().size()) { - GXF_LOG_ERROR("invalid image grid."); - return GXF_FAILURE; - } - std::copy(regions_width_.get().begin(), regions_width_.get().end(), image_grid_.regionWidth.begin()); - std::copy(regions_height_.get().begin(), regions_height_.get().end(), image_grid_.regionHeight.begin()); - std::copy(horizontal_intervals_.get().begin(), horizontal_intervals_.get().end(), image_grid_.horizInterval.begin()); - std::copy(vertical_intervals_.get().begin(), vertical_intervals_.get().end(), image_grid_.vertInterval.begin()); - output_shape_.x = static_cast( - std::accumulate(image_grid_.regionWidth.begin(), image_grid_.regionWidth.end(), 0)); - output_shape_.y = static_cast( - std::accumulate(image_grid_.regionHeight.begin(), image_grid_.regionHeight.end(), 0)); - - // Generate Image Warp if possible - if (input_camera_model_.try_get() && reference_frame_.try_get()) { - input_camera_info_ = {input_camera_model_.try_get().value()->getCameraIntrinsics(), - reference_frame_.try_get().value()->getCameraExtrinsics(), - input_camera_model_.try_get().value()->getDistortionModel()}; - - output_camera_intrinsics_ = output_camera_model_.try_get() - ? output_camera_model_.try_get().value()->getCameraIntrinsics() - : input_camera_info_.intrinsic; - - auto err_code = stream_->getStream()->GenerateWarpFromCameraModel(image_warp_, image_grid_, input_camera_info_, - output_camera_intrinsics_); - if (err_code != ::cvcore::make_error_condition(::cvcore::ErrorCode::SUCCESS)) { - GXF_LOG_ERROR("image warp creation failed."); - return GXF_FAILURE; - } - } - - return GXF_SUCCESS; -} - -gxf_result_t UndistortBase::stop() { - auto err_code = stream_->getStream()->DestroyWarp(image_warp_); - if (err_code != ::cvcore::make_error_condition(::cvcore::ErrorCode::SUCCESS)) { - GXF_LOG_ERROR("image warp de-allocation failed."); - return GXF_FAILURE; - } - return GXF_SUCCESS; -} - -gxf_result_t UndistortBase::registerInterface(gxf::Registrar* registrar) { - gxf::Expected result; - - result &= registrar->parameter(input_camera_model_, "input_camera_model", "", "", - gxf::Registrar::NoDefaultParameter(), GXF_PARAMETER_FLAGS_OPTIONAL); - result &= registrar->parameter(reference_frame_, "reference_frame", "", "", gxf::Registrar::NoDefaultParameter(), - GXF_PARAMETER_FLAGS_OPTIONAL); - result &= registrar->parameter(output_camera_model_, "output_camera_model", "", "", - gxf::Registrar::NoDefaultParameter(), GXF_PARAMETER_FLAGS_OPTIONAL); - result &= registrar->parameter(regions_width_, "regions_width"); - result &= registrar->parameter(regions_height_, "regions_height"); - result &= registrar->parameter(horizontal_intervals_, "horizontal_intervals"); - result &= registrar->parameter(vertical_intervals_, "vertical_intervals"); - result &= registrar->parameter(interp_type_, "interp_type"); - result &= registrar->parameter(border_type_, "border_type"); - result &= registrar->parameter(receiver_, "receiver"); - result &= registrar->parameter(transmitter_, "transmitter"); - result &= registrar->parameter(pool_, "pool"); - result &= registrar->parameter(stream_, "stream"); - result &= registrar->parameter(stream_pool_, "stream_pool", "cuda stream pool", "cuda stream pool object", - gxf::Registrar::NoDefaultParameter(), GXF_PARAMETER_FLAGS_OPTIONAL); - result &= registrar->parameter(input_adapter_, "input_adapter"); - result &= registrar->parameter(output_adapter_, "output_adapter"); - result &= registrar->parameter(input_name_, "input_name", "input name", "input tensor name", - gxf::Registrar::NoDefaultParameter(), GXF_PARAMETER_FLAGS_OPTIONAL); - result &= registrar->parameter(output_name_, "output_name", "output name", "output tensor name", - gxf::Registrar::NoDefaultParameter(), GXF_PARAMETER_FLAGS_OPTIONAL); - - return gxf::ToResultCode(result); -} - -gxf::Expected UndistortBase::doInferOutputInfo(gxf::Entity& input) { - // Check if the input distortion type is Perpective - auto maybe_camera_message = input.get(); - if (maybe_camera_message) { - no_op_ = maybe_camera_message.value()->distortion_type == gxf::DistortionType::Perspective; - } - // Output size may vary, but the format must be the same - return ImageInfo{input_info_.type, static_cast(output_shape_.x), static_cast(output_shape_.y), - input_info_.is_cpu}; -} - -gxf_result_t UndistortBase::doUpdateCameraMessage(gxf::Handle& output, - gxf::Handle& input) { - *output = *input; - (*output).distortion_type = gxf::DistortionType::Perspective; - for (size_t i = 0; i < gxf::CameraModel::kMaxDistortionCoefficients; i++) { - (*output).distortion_coefficients[i] = 0.; - } - (*output).dimensions = output_shape_; - (*output).focal_length.x = output_camera_intrinsics_.fx(); - (*output).focal_length.y = output_camera_intrinsics_.fy(); - (*output).principal_point.x = output_camera_intrinsics_.cx(); - (*output).principal_point.y = output_camera_intrinsics_.cy(); - (*output).skew_value = output_camera_intrinsics_.skew(); - return GXF_SUCCESS; -} - -#define DEFINE_UNDISTORT(INPUT_TYPE) \ - if (input_info_.type == INPUT_TYPE) { \ - return detail::UndistortImpl(output, input, output_info_, input_info_, output_name, input_name, \ - stream_.get(), output_adapter_.get(), input_adapter_.get(), pool_.get(), \ - image_warp_, interp.value(), border.value()); \ - } - -gxf_result_t UndistortBase::doExecute(gxf::Entity& output, gxf::Entity& input, cudaStream_t stream, - const char* output_name, const char* input_name) { - GXF_LOG_INFO("execute undistort."); - auto maybe_camera_message = input.get(); - auto maybe_pose3d_message = input.get(); - if (!maybe_camera_message) { - if (image_warp_ == nullptr) { - GXF_LOG_ERROR("no camera information found."); - return GXF_FAILURE; - } - } else { - auto maybe_intrinsics = detail::GetIntrinsicsFromMessage(maybe_camera_message.value()); - auto maybe_distortions = detail::GetDistortionsFromMessage(maybe_camera_message.value()); - if (!maybe_intrinsics || !maybe_distortions) { - return GXF_FAILURE; - } - const auto& new_intrinsics = maybe_intrinsics.value(); - const auto& new_distortions = maybe_distortions.value(); - - auto new_extrinsics = maybe_pose3d_message ? detail::GetExtrinsicsFromMessage(maybe_pose3d_message.value()).value() - : ::cvcore::CameraExtrinsics(); - - const bool reset = image_warp_ == nullptr || new_intrinsics != input_camera_info_.intrinsic || - new_distortions != input_camera_info_.distortion || - new_extrinsics != input_camera_info_.extrinsic; - - if (reset) { - auto new_width = static_cast(output_shape_.x); - auto new_height = static_cast(output_shape_.y); - - // These two parameters (width_scale and height_scale) can be - // used to determine a crop or pad regime depending on which dimension to - // preserve in the case of keep_aspect ratio. In this case, we assume - // always_crop=True, or that we will always use the largest dimension - // change. - auto width_scale = new_width / static_cast(input_info_.width); - auto height_scale = new_height / static_cast(input_info_.height); - auto scale = std::max({width_scale, height_scale}); // Always crop - - input_camera_info_ = {new_intrinsics, new_extrinsics, new_distortions}; - output_camera_intrinsics_ = new_intrinsics; - output_camera_intrinsics_.m_intrinsics[0][0] = scale * new_intrinsics.m_intrinsics[0][0]; - output_camera_intrinsics_.m_intrinsics[0][1] = scale * new_intrinsics.m_intrinsics[0][1]; - output_camera_intrinsics_.m_intrinsics[1][1] = scale * new_intrinsics.m_intrinsics[1][1]; - output_camera_intrinsics_.m_intrinsics[0][2] = scale * new_intrinsics.m_intrinsics[0][2]; - output_camera_intrinsics_.m_intrinsics[1][2] = scale * new_intrinsics.m_intrinsics[1][2]; - - auto err_code = stream_->getStream()->GenerateWarpFromCameraModel(image_warp_, image_grid_, input_camera_info_, - output_camera_intrinsics_); - if (err_code != ::cvcore::make_error_condition(::cvcore::ErrorCode::SUCCESS)) { - GXF_LOG_ERROR("image warp creation failed."); - return GXF_FAILURE; - } - } - } - - auto interp = GetInterpolationType(interp_type_); - if (!interp) { - return interp.error(); - } - auto border = GetBorderType(border_type_); - if (!border) { - return border.error(); - } - - // Run the image undistortion operation - DEFINE_UNDISTORT(::cvcore::ImageType::BGR_U8); - DEFINE_UNDISTORT(::cvcore::ImageType::RGB_U8); - DEFINE_UNDISTORT(::cvcore::ImageType::NV12); - DEFINE_UNDISTORT(::cvcore::ImageType::NV24); - - // Return error code for unsupported type - GXF_LOG_ERROR("invalid input/output type for image undistort."); - return GXF_FAILURE; -} - -} // namespace tensor_ops -} // namespace cvcore -} // namespace nvidia diff --git a/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/Undistort.hpp b/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/Undistort.hpp deleted file mode 100644 index 377f621..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/Undistort.hpp +++ /dev/null @@ -1,69 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 -#ifndef NVIDIA_CVCORE_UNDISTORT_HPP -#define NVIDIA_CVCORE_UNDISTORT_HPP - -#include "CameraModel.hpp" -#include "Frame3D.hpp" -#include "TensorOperator.hpp" - -namespace nvidia { -namespace cvcore { -namespace tensor_ops { - -// Undistort operator. -class UndistortBase : public TensorOperator { -public: - virtual ~UndistortBase() {} - - gxf_result_t start() override final; - gxf_result_t stop() override final; - - gxf_result_t registerInterface(gxf::Registrar* registrar) override; - -private: - gxf::Expected doInferOutputInfo(gxf::Entity& input) override final; - gxf_result_t doUpdateCameraMessage(gxf::Handle& output, - gxf::Handle& input) override final; - gxf_result_t doExecute(gxf::Entity& output, gxf::Entity& input, cudaStream_t stream, const char* output_name, - const char* input_name) override final; - - gxf::Parameter> input_camera_model_; - gxf::Parameter> reference_frame_; - gxf::Parameter> output_camera_model_; - gxf::Parameter> regions_width_; - gxf::Parameter> regions_height_; - gxf::Parameter> horizontal_intervals_; - gxf::Parameter> vertical_intervals_; - gxf::Parameter interp_type_; - gxf::Parameter border_type_; - - ::cvcore::tensor_ops::ImageGrid image_grid_; - ::cvcore::tensor_ops::ImageWarp image_warp_; - gxf::Vector2u output_shape_; - - ::cvcore::CameraModel input_camera_info_; - ::cvcore::CameraIntrinsics output_camera_intrinsics_; -}; - -class StreamUndistort : public UndistortBase {}; - -} // namespace tensor_ops -} // namespace cvcore -} // namespace nvidia - -#endif diff --git a/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/detail/ImageAdapterTensorImpl.cpp b/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/detail/ImageAdapterTensorImpl.cpp deleted file mode 100644 index 37099fc..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/detail/ImageAdapterTensorImpl.cpp +++ /dev/null @@ -1,105 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 -#include "ImageAdapterTensorImpl.hpp" - -namespace nvidia { -namespace cvcore { -namespace tensor_ops { -namespace detail { - -gxf::Expected> GetHWCIndices(const ::cvcore::ImageType type) { - switch (type) { - case ::cvcore::ImageType::Y_U8: - case ::cvcore::ImageType::Y_U16: - case ::cvcore::ImageType::Y_F32: - case ::cvcore::ImageType::RGB_U8: - case ::cvcore::ImageType::BGR_U8: - case ::cvcore::ImageType::RGB_U16: - case ::cvcore::ImageType::BGR_U16: - case ::cvcore::ImageType::RGB_F32: - case ::cvcore::ImageType::BGR_F32: { - return std::make_tuple(0, 1, 2); - } - case ::cvcore::ImageType::PLANAR_RGB_U8: - case ::cvcore::ImageType::PLANAR_BGR_U8: - case ::cvcore::ImageType::PLANAR_RGB_U16: - case ::cvcore::ImageType::PLANAR_BGR_U16: - case ::cvcore::ImageType::PLANAR_RGB_F32: - case ::cvcore::ImageType::PLANAR_BGR_F32: { - return std::make_tuple(1, 2, 0); - } - default: { - GXF_LOG_ERROR("invalid image type."); - return gxf::Unexpected{GXF_FAILURE}; - } - } -} - -gxf::Expected GetPrimitiveType(const ::cvcore::ImageType image_type) { - switch (image_type) { - case ::cvcore::ImageType::Y_U8: - case ::cvcore::ImageType::RGB_U8: - case ::cvcore::ImageType::BGR_U8: - case ::cvcore::ImageType::PLANAR_RGB_U8: - case ::cvcore::ImageType::PLANAR_BGR_U8: { - return gxf::PrimitiveType::kUnsigned8; - } - case ::cvcore::ImageType::Y_U16: - case ::cvcore::ImageType::RGB_U16: - case ::cvcore::ImageType::BGR_U16: - case ::cvcore::ImageType::PLANAR_RGB_U16: - case ::cvcore::ImageType::PLANAR_BGR_U16: { - return gxf::PrimitiveType::kUnsigned16; - } - case ::cvcore::ImageType::Y_F32: - case ::cvcore::ImageType::RGB_F32: - case ::cvcore::ImageType::BGR_F32: - case ::cvcore::ImageType::PLANAR_RGB_F32: - case ::cvcore::ImageType::PLANAR_BGR_F32: { - return gxf::PrimitiveType::kFloat32; - } - default: { - GXF_LOG_ERROR("invalid image type."); - return gxf::Unexpected{GXF_FAILURE}; - } - } -} - -gxf::Expected GetTensorInfo(gxf::Handle tensor, const ::cvcore::ImageType type) { - const auto& shape = tensor->shape(); - const auto rank = tensor->rank(); - const auto storage_type = tensor->storage_type(); - - if (rank != 3) { - GXF_LOG_ERROR("unexpected tensor shape."); - return gxf::Unexpected{GXF_FAILURE}; - } - - const auto indices = GetHWCIndices(type); - if (!indices) { - return gxf::Unexpected{GXF_FAILURE}; - } - const size_t width = shape.dimension(std::get<1>(indices.value())); - const size_t height = shape.dimension(std::get<0>(indices.value())); - - return ImageInfo{type, width, height, storage_type != gxf::MemoryStorageType::kDevice}; -} - -} // namespace detail -} // namespace tensor_ops -} // namespace cvcore -} // namespace nvidia diff --git a/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/detail/ImageAdapterTensorImpl.hpp b/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/detail/ImageAdapterTensorImpl.hpp deleted file mode 100644 index 45a060a..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/detail/ImageAdapterTensorImpl.hpp +++ /dev/null @@ -1,105 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 -#ifndef NVIDIA_CVCORE_TENSOR_ADAPTER_HPP -#define NVIDIA_CVCORE_TENSOR_ADAPTER_HPP - -#include "../ImageUtils.hpp" - -#include - -#include "cv/core/Image.h" -#include "gxf/std/allocator.hpp" -#include "gxf/std/tensor.hpp" - -namespace nvidia { -namespace cvcore { -namespace tensor_ops { -namespace detail { - -gxf::Expected> GetHWCIndices(const ::cvcore::ImageType type); - -gxf::Expected GetPrimitiveType(const ::cvcore::ImageType image_type); - -gxf::Expected GetTensorInfo(gxf::Handle tensor, const ::cvcore::ImageType type); - -template<::cvcore::ImageType T, - typename std::enable_if::type* = nullptr> -gxf::Expected<::cvcore::Image> WrapImageFromTensor(gxf::Handle tensor) { - const auto info = GetTensorInfo(tensor, T); - if (!info) { - return gxf::Unexpected{GXF_FAILURE}; - } - using D = typename ::cvcore::detail::ChannelTypeToNative<::cvcore::ImageTraits::CT>::Type; - auto pointer = tensor->data(); - if (!pointer) { - return gxf::Unexpected{GXF_FAILURE}; - } - const size_t stride = tensor->stride(std::get<0>(GetHWCIndices(T).value())); - return ::cvcore::Image(info.value().width, info.value().height, stride, pointer.value(), info.value().is_cpu); -} - -template<::cvcore::ImageType T, - typename std::enable_if::type* = nullptr> -gxf::Expected<::cvcore::Image> WrapImageFromTensor(gxf::Handle tensor) { - GXF_LOG_ERROR("NV12/NV24 not supported for gxf::Tensor"); - return gxf::Unexpected{GXF_FAILURE}; -} - -template<::cvcore::ImageType T, - typename std::enable_if::type* = nullptr> -gxf_result_t AllocateTensor(gxf::Handle tensor, size_t width, size_t height, - gxf::Handle allocator, bool is_cpu, bool allocate_pitch_linear) { - const auto primitive_type = GetPrimitiveType(T); - if (!primitive_type) { - return primitive_type.error(); - } - - const auto indices = GetHWCIndices(T); - if (!indices) { - return GXF_FAILURE; - } - std::array dims; - dims[std::get<0>(indices.value())] = height; - dims[std::get<1>(indices.value())] = width; - dims[std::get<2>(indices.value())] = ::cvcore::detail::ChannelToCount<::cvcore::ImageTraits::CC>(); - const gxf::Shape shape(dims, 3); - - auto result = - tensor->reshapeCustom(shape, primitive_type.value(), gxf::PrimitiveTypeSize(primitive_type.value()), - gxf::Unexpected{GXF_UNINITIALIZED_VALUE}, - is_cpu ? gxf::MemoryStorageType::kHost : gxf::MemoryStorageType::kDevice, allocator); - if (!result) { - GXF_LOG_ERROR("reshape tensor failed."); - return GXF_FAILURE; - } - return GXF_SUCCESS; -} - -template<::cvcore::ImageType T, - typename std::enable_if::type* = nullptr> -gxf_result_t AllocateTensor(gxf::Handle tensor, size_t width, size_t height, - gxf::Handle allocator, bool is_cpu, bool allocate_pitch_linear) { - GXF_LOG_ERROR("NV12/NV24 not supported for gxf::Tensor"); - return GXF_FAILURE; -} - -} // namespace detail -} // namespace tensor_ops -} // namespace cvcore -} // namespace nvidia - -#endif diff --git a/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/detail/ImageAdapterVideoBufferImpl.cpp b/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/detail/ImageAdapterVideoBufferImpl.cpp deleted file mode 100644 index ac257b4..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/detail/ImageAdapterVideoBufferImpl.cpp +++ /dev/null @@ -1,88 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 -#include "ImageAdapterVideoBufferImpl.hpp" - -namespace nvidia { -namespace cvcore { -namespace tensor_ops { -namespace detail { - -gxf::Expected<::cvcore::ImageType> GetImageTypeFromVideoFormat(const gxf::VideoFormat format) { - switch (format) { - case gxf::VideoFormat::GXF_VIDEO_FORMAT_NV12: - case gxf::VideoFormat::GXF_VIDEO_FORMAT_NV12_ER: { - return ::cvcore::ImageType::NV12; - } - case gxf::VideoFormat::GXF_VIDEO_FORMAT_NV24: - case gxf::VideoFormat::GXF_VIDEO_FORMAT_NV24_ER: { - return ::cvcore::ImageType::NV24; - } - case gxf::VideoFormat::GXF_VIDEO_FORMAT_RGBA: { - return ::cvcore::ImageType::RGBA_U8; - } - case gxf::VideoFormat::GXF_VIDEO_FORMAT_RGB: { - return ::cvcore::ImageType::RGB_U8; - } - case gxf::VideoFormat::GXF_VIDEO_FORMAT_RGB32: { - return ::cvcore::ImageType::RGB_F32; - } - case gxf::VideoFormat::GXF_VIDEO_FORMAT_BGR: { - return ::cvcore::ImageType::BGR_U8; - } - case gxf::VideoFormat::GXF_VIDEO_FORMAT_BGR32: { - return ::cvcore::ImageType::BGR_F32; - } - case gxf::VideoFormat::GXF_VIDEO_FORMAT_R8_G8_B8: { - return ::cvcore::ImageType::PLANAR_RGB_U8; - } - case gxf::VideoFormat::GXF_VIDEO_FORMAT_R32_G32_B32: { - return ::cvcore::ImageType::PLANAR_RGB_F32; - } - case gxf::VideoFormat::GXF_VIDEO_FORMAT_B8_G8_R8: { - return ::cvcore::ImageType::PLANAR_BGR_U8; - } - case gxf::VideoFormat::GXF_VIDEO_FORMAT_B32_G32_R32: { - return ::cvcore::ImageType::PLANAR_BGR_F32; - } - case gxf::VideoFormat::GXF_VIDEO_FORMAT_GRAY: { - return ::cvcore::ImageType::Y_U8; - } - case gxf::VideoFormat::GXF_VIDEO_FORMAT_GRAY32: { - return ::cvcore::ImageType::Y_F32; - } - default: { - GXF_LOG_ERROR("invalid video format."); - return gxf::Unexpected{GXF_FAILURE}; - } - } -} - -gxf::Expected GetVideoBufferInfo(gxf::Handle video_buffer) { - const auto buffer_info = video_buffer->video_frame_info(); - const auto storage_type = video_buffer->storage_type(); - auto image_type = GetImageTypeFromVideoFormat(buffer_info.color_format); - if (!image_type) { - return gxf::Unexpected{GXF_FAILURE}; - } - return ImageInfo{image_type.value(), buffer_info.width, buffer_info.height, - storage_type != gxf::MemoryStorageType::kDevice}; -} - -} // namespace detail -} // namespace tensor_ops -} // namespace cvcore -} // namespace nvidia diff --git a/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/detail/ImageAdapterVideoBufferImpl.hpp b/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/detail/ImageAdapterVideoBufferImpl.hpp deleted file mode 100644 index 6664640..0000000 --- a/isaac_ros_bi3d/gxf/bi3d/extensions/tensor_ops/detail/ImageAdapterVideoBufferImpl.hpp +++ /dev/null @@ -1,294 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 -#ifndef NVIDIA_CVCORE_VIDEO_BUFFER_ADAPTER_HPP -#define NVIDIA_CVCORE_VIDEO_BUFFER_ADAPTER_HPP - -#include "../ImageUtils.hpp" - -#include "gxf/multimedia/video.hpp" -#include "gxf/std/allocator.hpp" - -#include "cv/core/Image.h" - -namespace nvidia { -namespace cvcore { -namespace tensor_ops { -namespace detail { - -gxf::Expected<::cvcore::ImageType> GetImageTypeFromVideoFormat(const gxf::VideoFormat format); - -gxf::Expected GetVideoBufferInfo(gxf::Handle video_buffer); - -template<::cvcore::ImageType T, - typename std::enable_if::type* = nullptr> -gxf::Expected<::cvcore::Image> WrapImageFromVideoBuffer(gxf::Handle video_buffer) { - const auto info = GetVideoBufferInfo(video_buffer); - if (!info) { - return gxf::Unexpected{GXF_FAILURE}; - } - using D = typename ::cvcore::detail::ChannelTypeToNative<::cvcore::ImageTraits::CT>::Type; - auto pointer = reinterpret_cast(video_buffer->pointer()); - if (!pointer) { - return gxf::Unexpected{GXF_FAILURE}; - } - const auto& color_planes = video_buffer->video_frame_info().color_planes; - return ::cvcore::Image(info.value().width, info.value().height, color_planes[0].stride, pointer, - info.value().is_cpu); -} - -template<::cvcore::ImageType T, - typename std::enable_if::type* = nullptr> -gxf::Expected<::cvcore::Image> WrapImageFromVideoBuffer(gxf::Handle video_buffer) { - const auto info = GetVideoBufferInfo(video_buffer); - if (!info) { - return gxf::Unexpected{GXF_FAILURE}; - } - // Note only U8 is supported in NV12/NV24 - auto pointer = reinterpret_cast(video_buffer->pointer()); - if (!pointer) { - return gxf::Unexpected{GXF_FAILURE}; - } - const auto& color_planes = video_buffer->video_frame_info().color_planes; - return ::cvcore::Image(info.value().width, info.value().height, color_planes[0].stride, color_planes[1].stride, - pointer, pointer + color_planes[1].offset, info.value().is_cpu); -} - -template<::cvcore::ImageType T> -struct ImageTypeToVideoFormat { - static constexpr gxf::VideoFormat format = gxf::VideoFormat::GXF_VIDEO_FORMAT_CUSTOM; -}; - -template<> -struct ImageTypeToVideoFormat<::cvcore::ImageType::NV12> { - static constexpr gxf::VideoFormat format = gxf::VideoFormat::GXF_VIDEO_FORMAT_NV12_ER; -}; - -template<> -struct ImageTypeToVideoFormat<::cvcore::ImageType::NV24> { - static constexpr gxf::VideoFormat format = gxf::VideoFormat::GXF_VIDEO_FORMAT_NV24_ER; -}; - -template<> -struct ImageTypeToVideoFormat<::cvcore::ImageType::RGBA_U8> { - static constexpr gxf::VideoFormat format = gxf::VideoFormat::GXF_VIDEO_FORMAT_RGBA; -}; - -template<> -struct ImageTypeToVideoFormat<::cvcore::ImageType::RGB_U8> { - static constexpr gxf::VideoFormat format = gxf::VideoFormat::GXF_VIDEO_FORMAT_RGB; -}; - -template<> -struct ImageTypeToVideoFormat<::cvcore::ImageType::RGB_F32> { - static constexpr gxf::VideoFormat format = gxf::VideoFormat::GXF_VIDEO_FORMAT_RGB32; -}; - -template<> -struct ImageTypeToVideoFormat<::cvcore::ImageType::BGR_U8> { - static constexpr gxf::VideoFormat format = gxf::VideoFormat::GXF_VIDEO_FORMAT_BGR; -}; - -template<> -struct ImageTypeToVideoFormat<::cvcore::ImageType::BGR_F32> { - static constexpr gxf::VideoFormat format = gxf::VideoFormat::GXF_VIDEO_FORMAT_BGR32; -}; - -template<> -struct ImageTypeToVideoFormat<::cvcore::ImageType::PLANAR_RGB_U8> { - static constexpr gxf::VideoFormat format = gxf::VideoFormat::GXF_VIDEO_FORMAT_R8_G8_B8; -}; - -template<> -struct ImageTypeToVideoFormat<::cvcore::ImageType::PLANAR_RGB_F32> { - static constexpr gxf::VideoFormat format = gxf::VideoFormat::GXF_VIDEO_FORMAT_R32_G32_B32; -}; - -template<> -struct ImageTypeToVideoFormat<::cvcore::ImageType::PLANAR_BGR_U8> { - static constexpr gxf::VideoFormat format = gxf::VideoFormat::GXF_VIDEO_FORMAT_B8_G8_R8; -}; - -template<> -struct ImageTypeToVideoFormat<::cvcore::ImageType::PLANAR_BGR_F32> { - static constexpr gxf::VideoFormat format = gxf::VideoFormat::GXF_VIDEO_FORMAT_B32_G32_R32; -}; - -template<> -struct ImageTypeToVideoFormat<::cvcore::ImageType::Y_U8> { - static constexpr gxf::VideoFormat format = gxf::VideoFormat::GXF_VIDEO_FORMAT_GRAY; -}; - -template<> -struct ImageTypeToVideoFormat<::cvcore::ImageType::Y_F32> { - static constexpr gxf::VideoFormat format = gxf::VideoFormat::GXF_VIDEO_FORMAT_GRAY32; -}; - -template<::cvcore::ImageType T> -struct DefaultNoPaddingColorPlanes {}; - -template<> -struct DefaultNoPaddingColorPlanes<::cvcore::ImageType::NV12> { - DefaultNoPaddingColorPlanes(size_t width) - : planes({gxf::ColorPlane("Y", 1, width), gxf::ColorPlane("UV", 2, width)}) {} - std::array planes; -}; - -template<> -struct DefaultNoPaddingColorPlanes<::cvcore::ImageType::NV24> { - DefaultNoPaddingColorPlanes(size_t width) - : planes({gxf::ColorPlane("Y", 1, width), gxf::ColorPlane("UV", 2, width * 2)}) {} - std::array planes; -}; - -template<> -struct DefaultNoPaddingColorPlanes<::cvcore::ImageType::RGBA_U8> { - DefaultNoPaddingColorPlanes(size_t width) - : planes({gxf::ColorPlane("RGBA", 4, width * 4)}) {} - std::array planes; -}; - -template<> -struct DefaultNoPaddingColorPlanes<::cvcore::ImageType::RGB_U8> { - DefaultNoPaddingColorPlanes(size_t width) - : planes({gxf::ColorPlane("RGB", 3, width * 3)}) {} - std::array planes; -}; - -template<> -struct DefaultNoPaddingColorPlanes<::cvcore::ImageType::RGB_F32> { - DefaultNoPaddingColorPlanes(size_t width) - : planes({gxf::ColorPlane("RGB", 12, width * 12)}) {} - std::array planes; -}; - -template<> -struct DefaultNoPaddingColorPlanes<::cvcore::ImageType::BGR_U8> { - DefaultNoPaddingColorPlanes(size_t width) - : planes({gxf::ColorPlane("BGR", 3, width * 3)}) {} - std::array planes; -}; - -template<> -struct DefaultNoPaddingColorPlanes<::cvcore::ImageType::BGR_F32> { - DefaultNoPaddingColorPlanes(size_t width) - : planes({gxf::ColorPlane("BGR", 12, width * 12)}) {} - std::array planes; -}; - -template<> -struct DefaultNoPaddingColorPlanes<::cvcore::ImageType::PLANAR_RGB_U8> { - DefaultNoPaddingColorPlanes(size_t width) - : planes({gxf::ColorPlane("R", 1, width), gxf::ColorPlane("G", 1, width), gxf::ColorPlane("B", 1, width)}) {} - std::array planes; -}; - -template<> -struct DefaultNoPaddingColorPlanes<::cvcore::ImageType::PLANAR_RGB_F32> { - DefaultNoPaddingColorPlanes(size_t width) - : planes( - {gxf::ColorPlane("R", 4, width * 4), gxf::ColorPlane("G", 4, width * 4), gxf::ColorPlane("B", 4, width * 4)}) {} - std::array planes; -}; - -template<> -struct DefaultNoPaddingColorPlanes<::cvcore::ImageType::PLANAR_BGR_U8> { - DefaultNoPaddingColorPlanes(size_t width) - : planes({gxf::ColorPlane("B", 1, width), gxf::ColorPlane("G", 1, width), gxf::ColorPlane("R", 1, width)}) {} - std::array planes; -}; - -template<> -struct DefaultNoPaddingColorPlanes<::cvcore::ImageType::PLANAR_BGR_F32> { - DefaultNoPaddingColorPlanes(size_t width) - : planes( - {gxf::ColorPlane("B", 4, width * 4), gxf::ColorPlane("G", 4, width * 4), gxf::ColorPlane("R", 4, width * 4)}) {} - std::array planes; -}; - -template<> -struct DefaultNoPaddingColorPlanes<::cvcore::ImageType::Y_U8> { - DefaultNoPaddingColorPlanes(size_t width) - : planes({gxf::ColorPlane("gray", 1, width)}) {} - std::array planes; -}; - -template<> -struct DefaultNoPaddingColorPlanes<::cvcore::ImageType::Y_F32> { - DefaultNoPaddingColorPlanes(size_t width) - : planes({gxf::ColorPlane("gray", 4, width * 4)}) {} - std::array planes; -}; - -// This include the list of image types that GXF supported so far -constexpr bool IsValidGXFImageType(const ::cvcore::ImageType type) { - return type == ::cvcore::ImageType::NV12 || type == ::cvcore::ImageType::NV24 || - type == ::cvcore::ImageType::RGBA_U8 || type == ::cvcore::ImageType::RGB_U8 || - type == ::cvcore::ImageType::BGR_U8 || type == ::cvcore::ImageType::RGB_F32 || - type == ::cvcore::ImageType::BGR_F32 || type == ::cvcore::ImageType::PLANAR_RGB_U8 || - type == ::cvcore::ImageType::PLANAR_BGR_U8 || type == ::cvcore::ImageType::PLANAR_RGB_F32 || - type == ::cvcore::ImageType::PLANAR_BGR_F32 || type == ::cvcore::ImageType::Y_U8 || - type == ::cvcore::ImageType::Y_F32; -} - -template<::cvcore::ImageType T, typename std::enable_if::type* = nullptr> -gxf_result_t AllocateVideoBuffer(gxf::Handle video_buffer, size_t width, size_t height, - gxf::Handle allocator, bool is_cpu, bool allocate_pitch_linear) { - if (width % 2 != 0 || height % 2 != 0) { - GXF_LOG_ERROR("image width/height must be even for creation of gxf::VideoBuffer"); - return GXF_FAILURE; - } - if (allocate_pitch_linear) { - auto result = video_buffer->resize::format>( - static_cast(width), static_cast(height), gxf::SurfaceLayout::GXF_SURFACE_LAYOUT_PITCH_LINEAR, - is_cpu ? gxf::MemoryStorageType::kHost : gxf::MemoryStorageType::kDevice, allocator); - - if (!result) { - GXF_LOG_ERROR("resize VideoBuffer failed."); - return GXF_FAILURE; - } - } else { - DefaultNoPaddingColorPlanes planes_trait(width); - gxf::VideoFormatSize::format> buffer_type_trait; - uint64_t size = buffer_type_trait.size(width, height, planes_trait.planes); - std::vector planes_filled{planes_trait.planes.begin(), planes_trait.planes.end()}; - gxf::VideoBufferInfo buffer_info{static_cast(width), static_cast(height), - ImageTypeToVideoFormat::format, planes_filled, - gxf::SurfaceLayout::GXF_SURFACE_LAYOUT_PITCH_LINEAR}; - auto result = video_buffer->resizeCustom( - buffer_info, size, is_cpu ? gxf::MemoryStorageType::kHost : gxf::MemoryStorageType::kDevice, allocator); - - if (!result) { - GXF_LOG_ERROR("custom resize VideoBuffer failed."); - return GXF_FAILURE; - } - } - return GXF_SUCCESS; -} - -template<::cvcore::ImageType T, typename std::enable_if::type* = nullptr> -gxf_result_t AllocateVideoBuffer(gxf::Handle video_buffer, size_t width, size_t height, - gxf::Handle allocator, bool is_cpu, bool allocate_pitch_linear) { - GXF_LOG_ERROR("image type not supported in gxf::VideoBuffer"); - return GXF_FAILURE; -} - -} // namespace detail -} // namespace tensor_ops -} // namespace cvcore -} // namespace nvidia - -#endif diff --git a/isaac_ros_bi3d/gxf/bi3d_postprocessor/CMakeLists.txt b/isaac_ros_bi3d/gxf/bi3d_postprocessor/CMakeLists.txt deleted file mode 100644 index 1ab2142..0000000 --- a/isaac_ros_bi3d/gxf/bi3d_postprocessor/CMakeLists.txt +++ /dev/null @@ -1,60 +0,0 @@ -# SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -# Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# SPDX-License-Identifier: Apache-2.0 - -project(gxf_bi3d_postprocessor LANGUAGES C CXX CUDA) - -# Dependencies -find_package(CUDAToolkit) -include(YamlCpp) -find_package(GXF ${ISAAC_ROS_GXF_VERSION} MODULE REQUIRED - COMPONENTS - core - cuda - multimedia - serialization - std -) - -# Create extension -add_library(gxf_bi3d_postprocessor SHARED - bi3d_postprocessor_extension.cpp - bi3d_postprocess/bi3d_message_splitter.cpp - bi3d_postprocess/bi3d_message_splitter.hpp - bi3d_postprocessor_utils.hpp - bi3d_postprocess/bi3d_postprocessor.cpp - bi3d_postprocess/bi3d_postprocessor.cu.cpp - bi3d_postprocess/bi3d_postprocessor.cu.hpp - bi3d_postprocess/bi3d_postprocessor.hpp -) - -# Mark as CUDA files with non-standard extensions -set_source_files_properties( - bi3d_postprocess/bi3d_postprocessor.cu.cpp - bi3d_postprocess/bi3d_postprocessor.cu.hpp - PROPERTIES - LANGUAGE CUDA -) - -set(CMAKE_INCLUDE_CURRENT_DIR TRUE) - -target_link_libraries(gxf_bi3d_postprocessor - PUBLIC - GXF::cuda - GXF::multimedia - GXF::std - yaml-cpp -) diff --git a/isaac_ros_bi3d/gxf/bi3d_postprocessor/bi3d_postprocess/bi3d_message_splitter.cpp b/isaac_ros_bi3d/gxf/bi3d_postprocessor/bi3d_postprocess/bi3d_message_splitter.cpp deleted file mode 100644 index 00ad8f7..0000000 --- a/isaac_ros_bi3d/gxf/bi3d_postprocessor/bi3d_postprocess/bi3d_message_splitter.cpp +++ /dev/null @@ -1,67 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -#include "gxf/std/parameter_parser_std.hpp" -#include "gxf/std/timestamp.hpp" -#include "gxf/multimedia/video.hpp" -#include "bi3d_message_splitter.hpp" - -namespace nvidia -{ -namespace isaac_ros -{ - -gxf_result_t Bi3DMessageSplitter::registerInterface(gxf::Registrar * registrar) noexcept -{ - gxf::Expected result; - result &= registrar->parameter( - receiver_, "receiver", "Receiver", - "Message from Bi3D post processor"); - result &= registrar->parameter( - disparity_image_transmitter_, "disparity_image_transmitter", "Disparity image transmitter", - "The collapsed output of Bi3D"); - result &= registrar->parameter( - disparity_values_transmitter_, "disparity_values_transmitter", "Disparity values transmitter", - "The disparity values used for Bi3D inference"); - return gxf::ToResultCode(result); -} - -gxf_result_t Bi3DMessageSplitter::tick() noexcept -{ - // Receive bi3d post processor output - auto bi3d_postprocessed_message = receiver_->receive(); - if (!bi3d_postprocessed_message) { - GXF_LOG_ERROR("Failed to get message"); - return gxf::ToResultCode(bi3d_postprocessed_message); - } - - // Publish message - auto result = disparity_image_transmitter_->publish(bi3d_postprocessed_message.value()); - if (!result) { - return gxf::ToResultCode(result); - } - result = disparity_values_transmitter_->publish(bi3d_postprocessed_message.value()); - if (!result) { - return gxf::ToResultCode(result); - } - - return GXF_SUCCESS; - -} - -} // namespace isaac_ros -} // namespace nvidia diff --git a/isaac_ros_bi3d/gxf/bi3d_postprocessor/bi3d_postprocess/bi3d_message_splitter.hpp b/isaac_ros_bi3d/gxf/bi3d_postprocessor/bi3d_postprocess/bi3d_message_splitter.hpp deleted file mode 100644 index b642f30..0000000 --- a/isaac_ros_bi3d/gxf/bi3d_postprocessor/bi3d_postprocess/bi3d_message_splitter.hpp +++ /dev/null @@ -1,50 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -#ifndef NVIDIA_ISAAC_ROS_EXTENSIONS_BI3D_MESSAGE_SPLITTER_HPP_ -#define NVIDIA_ISAAC_ROS_EXTENSIONS_BI3D_MESSAGE_SPLITTER_HPP_ - -#include "gxf/std/allocator.hpp" -#include "gxf/std/codelet.hpp" -#include "gxf/std/receiver.hpp" -#include "gxf/std/transmitter.hpp" - - -namespace nvidia -{ -namespace isaac_ros -{ - -// GXF codelet that takes the postprocessed output of Bi3D and splits it into disparity image and disparity values -class Bi3DMessageSplitter : public gxf::Codelet -{ -public: - gxf_result_t registerInterface(gxf::Registrar * registrar) noexcept override; - gxf_result_t start() noexcept override {return GXF_SUCCESS;} - gxf_result_t tick() noexcept override; - gxf_result_t stop() noexcept override {return GXF_SUCCESS;} - -private: - gxf::Parameter> receiver_; - gxf::Parameter> disparity_image_transmitter_; - gxf::Parameter> disparity_values_transmitter_; -}; - -} // namespace isaac_ros -} // namespace nvidia - -#endif // NVIDIA_ISAAC_ROS_EXTENSIONS_BI3D_MESSAGE_SPLITTER_HPP_ diff --git a/isaac_ros_bi3d/gxf/bi3d_postprocessor/bi3d_postprocess/bi3d_postprocessor.cpp b/isaac_ros_bi3d/gxf/bi3d_postprocessor/bi3d_postprocess/bi3d_postprocessor.cpp deleted file mode 100644 index 740eeba..0000000 --- a/isaac_ros_bi3d/gxf/bi3d_postprocessor/bi3d_postprocess/bi3d_postprocessor.cpp +++ /dev/null @@ -1,178 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -#include "gxf/std/parameter_parser_std.hpp" -#include "gxf/std/timestamp.hpp" -#include "bi3d_postprocessor.hpp" -#include "bi3d_postprocessor_utils.hpp" -#include "bi3d_postprocessor.cu.hpp" - - -namespace nvidia -{ -namespace isaac_ros -{ - -gxf_result_t Bi3DPostprocessor::registerInterface(gxf::Registrar * registrar) noexcept -{ - gxf::Expected result; - result &= registrar->parameter( - bi3d_receiver_, "bi3d_receiver", "Bi3D input", - "Tensor from Bi3D"); - result &= registrar->parameter( - output_transmitter_, "output_transmitter", "Output transmitter", - "The collapsed output of Bi3D"); - result &= registrar->parameter( - pool_, "pool", "Pool", - "Allocator instance for output videobuffer"); - result &= registrar->parameter( - disparity_values_, "disparity_values", "Disparity values", - "Fixed disparity values used for Bi3D inference", - gxf::Registrar::NoDefaultParameter(), GXF_PARAMETER_FLAGS_OPTIONAL); - result &= registrar->parameter( - disparity_tensor_name_, "disparity_tensor_name", "Disparity tensor name", - "Name of the disparity tensor from Bi3D"); - result &= registrar->parameter( - disparity_values_tensor_name_, "disparity_values_tensor_name", "Disparity values tensor name", - "Name of the (dynamic) disparity values tensor from Bi3D", - gxf::Registrar::NoDefaultParameter(), GXF_PARAMETER_FLAGS_OPTIONAL); - return gxf::ToResultCode(result); -} - -gxf_result_t Bi3DPostprocessor::start() noexcept -{ - auto fixed_disparity_values = disparity_values_.try_get(); - auto disparity_values_name = disparity_values_tensor_name_.try_get(); - if (fixed_disparity_values && disparity_values_name) { - GXF_LOG_WARNING( - "Both fixed disparity values and dynamic disparity values given. Fixed disparity values will be used."); - } - return GXF_SUCCESS; -} - -gxf_result_t Bi3DPostprocessor::tick() noexcept -{ - // Receive bi3d output - auto bi3d_message = bi3d_receiver_->receive(); - if (!bi3d_message) { - GXF_LOG_ERROR("Failed to get message"); - return gxf::ToResultCode(bi3d_message); - } - - // Create output message - auto out_message = gxf::Entity::New(context()); - if (!out_message) { - GXF_LOG_ERROR("Failed to allocate message"); - return gxf::ToResultCode(out_message); - } - - // Get Bi3D disparity tensor - auto bi3d_disparity_tensor = bi3d_message->get(disparity_tensor_name_.get().c_str()); - if (!bi3d_disparity_tensor) { - GXF_LOG_ERROR("Failed to get disparity image tensor"); - return gxf::ToResultCode(bi3d_disparity_tensor); - } - if (!bi3d_disparity_tensor.value()->data()) { - GXF_LOG_ERROR("Failed to get pointer to bi3d disparity data"); - return gxf::ToResultCode(bi3d_disparity_tensor.value()->data()); - } - const float * bi3d_disparity_data_ptr = bi3d_disparity_tensor.value()->data().value(); - - - // Get disparity values used for Bi3D inference - int disparity_count; - const int * disparity_values_data; - - auto fixed_disparity_values = disparity_values_.try_get(); - if (fixed_disparity_values) { - disparity_count = fixed_disparity_values.value().size(); - disparity_values_data = fixed_disparity_values.value().data(); - } else { - auto disparity_values_name = disparity_values_tensor_name_.try_get(); - if (!disparity_values_name) { - GXF_LOG_ERROR("Neither dynamic nor fixed disparity values specified"); - return GXF_FAILURE; - } - auto bi3d_disparity_value_tensor = bi3d_message->get( - disparity_values_name.value().c_str()); - if (!bi3d_disparity_value_tensor) { - GXF_LOG_ERROR("Failed to get disparity values tensor"); - return gxf::ToResultCode(bi3d_disparity_value_tensor); - } - disparity_count = bi3d_disparity_value_tensor.value()->element_count(); - disparity_values_data = bi3d_disparity_value_tensor.value()->data().value(); - - // Add dynamic disparity value tensor to output message - auto forwarded_disp_values = out_message.value().add( - bi3d_disparity_value_tensor->name()); - *forwarded_disp_values.value() = std::move(*bi3d_disparity_value_tensor.value()); - } - - // Get dimensions of tensor - gxf::Shape dims = bi3d_disparity_tensor.value()->shape(); - const int image_height = dims.dimension(1); - const int image_width = dims.dimension(2); - const int image_size = dims.dimension(1) * dims.dimension(2); - - // Create video buffer - auto out_video_buffer = out_message.value().add(); - if (!out_video_buffer) { - GXF_LOG_ERROR("Failed to allocate output video buffer"); - return gxf::ToResultCode(out_video_buffer); - } - - // Allocate video buffer on device (unpadded gray32) - auto maybe_allocation = AllocateVideoBuffer( - out_video_buffer.value(), image_width, image_height, gxf::MemoryStorageType::kDevice, - pool_.get()); - if (!maybe_allocation) { - GXF_LOG_ERROR("Failed to allocate output video buffer's memory."); - return gxf::ToResultCode(maybe_allocation); - } - - cudaMemset( - out_video_buffer.value()->pointer(), 0, - image_size * bi3d_disparity_tensor.value()->bytes_per_element()); - - for (int i = 0; i < disparity_count; i++) { - cuda_postprocess( - bi3d_disparity_data_ptr + (i * image_size), - (float *)out_video_buffer.value()->pointer(), disparity_values_data[i], image_height, - image_width); - } - - // Add timestamp - std::string timestamp_name{"timestamp"}; - auto maybe_bi3d_timestamp = bi3d_message->get(); - if (!maybe_bi3d_timestamp) { - GXF_LOG_ERROR("Failed to get a timestamp from Bi3D output"); - } - auto out_timestamp = out_message.value().add(timestamp_name.c_str()); - if (!out_timestamp) {return GXF_FAILURE;} - *out_timestamp.value() = *maybe_bi3d_timestamp.value(); - - // Publish message - auto result = output_transmitter_->publish(out_message.value()); - if (!result) { - return gxf::ToResultCode(result); - } - - return GXF_SUCCESS; -} - -} // namespace isaac_ros -} // namespace nvidia diff --git a/isaac_ros_bi3d/gxf/bi3d_postprocessor/bi3d_postprocess/bi3d_postprocessor.cu.cpp b/isaac_ros_bi3d/gxf/bi3d_postprocessor/bi3d_postprocess/bi3d_postprocessor.cu.cpp deleted file mode 100644 index 931525b..0000000 --- a/isaac_ros_bi3d/gxf/bi3d_postprocessor/bi3d_postprocess/bi3d_postprocessor.cu.cpp +++ /dev/null @@ -1,54 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 -#include "bi3d_postprocessor.cu.hpp" - -namespace nvidia -{ -namespace isaac_ros -{ - -__global__ void postprocessing_kernel( - const float * input, float * output, int disparity, - int imageHeight, int imageWidth) -{ - const uint32_t x = blockIdx.x * blockDim.x + threadIdx.x; - const uint32_t y = blockIdx.y * blockDim.y + threadIdx.y; - const uint32_t index = y * imageWidth + x; - if (x < imageWidth && y < imageHeight) { - if (input[index] == 1.0) { - output[index] = input[index] * disparity; - } - } -} - -uint16_t ceil_div(uint16_t numerator, uint16_t denominator) -{ - uint32_t accumulator = numerator + denominator - 1; - return accumulator / denominator; -} - -void cuda_postprocess( - const float * input, float * output, int disparity, int imageHeight, - int imageWidth) -{ - dim3 block(16, 16); - dim3 grid(ceil_div(imageWidth, 16), ceil_div(imageHeight, 16), 1); - postprocessing_kernel << < grid, block >> > (input, output, disparity, imageHeight, imageWidth); -} - -} // namespace isaac_ros -} // namespace nvidia diff --git a/isaac_ros_bi3d/gxf/bi3d_postprocessor/bi3d_postprocessor_extension.cpp b/isaac_ros_bi3d/gxf/bi3d_postprocessor/bi3d_postprocessor_extension.cpp deleted file mode 100644 index ebe5ee6..0000000 --- a/isaac_ros_bi3d/gxf/bi3d_postprocessor/bi3d_postprocessor_extension.cpp +++ /dev/null @@ -1,45 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 -#include - -#include "gxf/core/gxf.h" -#include "gxf/std/extension_factory_helper.hpp" -#include "bi3d_postprocess/bi3d_postprocessor.hpp" -#include "bi3d_postprocess/bi3d_message_splitter.hpp" - -extern "C" { - -GXF_EXT_FACTORY_BEGIN() - -GXF_EXT_FACTORY_SET_INFO( - 0xb764a9fce34d11ec, 0x8fea0242ac120002, "Bi3DPostprocessorExtension", - "Bi3D Post Processing GXF extension", - "NVIDIA", "1.0.0", "LICENSE"); - -GXF_EXT_FACTORY_ADD( - 0xa8a24ecde2544d6a, 0x9a4b81cdb71085d6, - nvidia::isaac_ros::Bi3DPostprocessor, nvidia::gxf::Codelet, - "Collapses Bi3D output tensorlist to single segmentation image"); - -GXF_EXT_FACTORY_ADD( - 0xa8a24ecde2564d6a, 0x9a4b89cdb71085d6, - nvidia::isaac_ros::Bi3DMessageSplitter, nvidia::gxf::Codelet, - "Splits Bi3D postprocessor output into disparity image and disparity values"); - -GXF_EXT_FACTORY_END() - -} // extern "C" diff --git a/isaac_ros_bi3d/gxf/bi3d_postprocessor/bi3d_postprocessor_utils.hpp b/isaac_ros_bi3d/gxf/bi3d_postprocessor/bi3d_postprocessor_utils.hpp deleted file mode 100644 index d6d64c5..0000000 --- a/isaac_ros_bi3d/gxf/bi3d_postprocessor/bi3d_postprocessor_utils.hpp +++ /dev/null @@ -1,87 +0,0 @@ -// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 -#ifndef NVIDIA_ISAAC_ROS_EXTENSIONS_BI3D_POSTPROCESSOR_UTILS_HPP_ -#define NVIDIA_ISAAC_ROS_EXTENSIONS_BI3D_POSTPROCESSOR_UTILS_HPP_ - -#include - -#include "gxf/multimedia/video.hpp" -#include "gxf/std/allocator.hpp" - -namespace nvidia -{ -namespace isaac_ros -{ - -template -inline constexpr uint8_t GetChannelSize(); - -template<> -inline constexpr uint8_t GetChannelSize() -{ - return 1; -} - -template<> -inline constexpr uint8_t GetChannelSize() -{ - return 4; -} - -template -inline const char * GetColorName(); - -template<> -inline const char * GetColorName() -{ - return "gray"; -} - -template<> -inline const char * GetColorName() -{ - return "gray"; -} - -template -gxf::Expected AllocateVideoBuffer( - gxf::Handle video_buffer, size_t width, - size_t height, gxf::MemoryStorageType memory_storage_type, - gxf::Handle allocator) -{ - if (width % 2 != 0 || height % 2 != 0) { - GXF_LOG_ERROR( - "Image width and height must be even, but received width: %zu, height: %zu", - width, height); - return gxf::Unexpected{GXF_FAILURE}; - } - - std::array planes{ - gxf::ColorPlane(GetColorName(), GetChannelSize(), GetChannelSize() * width)}; - gxf::VideoFormatSize video_format_size; - const uint64_t size = video_format_size.size(width, height, planes); - const std::vector planes_filled{planes.begin(), planes.end()}; - const gxf::VideoBufferInfo buffer_info{static_cast(width), - static_cast(height), T, planes_filled, - gxf::SurfaceLayout::GXF_SURFACE_LAYOUT_PITCH_LINEAR}; - return video_buffer->resizeCustom(buffer_info, size, memory_storage_type, allocator); -} - -} // namespace isaac_ros -} // namespace nvidia - -#endif // NVIDIA_ISAAC_ROS_EXTENSIONS_BI3D_POSTPROCESSOR_UTILS_HPP_ diff --git a/isaac_ros_bi3d/gxf/extensions/bi3d/bi3d.cpp b/isaac_ros_bi3d/gxf/extensions/bi3d/bi3d.cpp new file mode 100644 index 0000000..38f0855 --- /dev/null +++ b/isaac_ros_bi3d/gxf/extensions/bi3d/bi3d.cpp @@ -0,0 +1,40 @@ +// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES +// Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 +#include "extensions/bi3d/components/bi3d_depthview.hpp" +#include "extensions/bi3d/components/bi3d_inference.hpp" +#include "extensions/bi3d/components/bi3d_postprocessor.hpp" +#include "gxf/std/extension_factory_helper.hpp" + +GXF_EXT_FACTORY_BEGIN() + +GXF_EXT_FACTORY_SET_INFO(0xce7c6985267a4ec7, 0xa073030e16e49f29, "Bi3D", + "Extension containing Bi3D related components", + "Isaac SDK", "2.0.0", "LICENSE"); + +GXF_EXT_FACTORY_ADD(0x9cfc86f71c7c4102, 0xade57d2d4faf6453, + nvidia::isaac::bi3d::DepthView, nvidia::gxf::Codelet, + "Bi3d depth map generator"); + +GXF_EXT_FACTORY_ADD(0x2eb361c045894aec, 0x831550ff5f177d87, + nvidia::isaac::bi3d::Bi3DPostprocessor, nvidia::gxf::Codelet, + "Bi3D post processor"); + +GXF_EXT_FACTORY_ADD(0xdcba0cf83a5340d2, 0x8404788350ad2324, + nvidia::isaac::Bi3DInference, nvidia::gxf::Codelet, + "Bi3D Inference"); + +GXF_EXT_FACTORY_END() diff --git a/isaac_ros_bi3d/gxf/extensions/bi3d/components/bi3d_depthview.cpp b/isaac_ros_bi3d/gxf/extensions/bi3d/components/bi3d_depthview.cpp new file mode 100644 index 0000000..52fda8e --- /dev/null +++ b/isaac_ros_bi3d/gxf/extensions/bi3d/components/bi3d_depthview.cpp @@ -0,0 +1,200 @@ +// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES +// Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 +#include "extensions/bi3d/components/bi3d_depthview.hpp" + +#include +#include +#include + +#include "engine/core/image/image.hpp" +#include "engine/core/tensor/tensor.hpp" +#include "engine/gems/image/io.hpp" +#include "engine/gems/image/utils.hpp" +#include "gxf/multimedia/video.hpp" + +namespace nvidia { +namespace isaac { +namespace bi3d { + +// Indicates Depth View Color Scheme. +enum DepthViewColorScheme { + DepthViewColorScheme_ErrorColor = 1, + DepthViewColorScheme_RedGreenBlack, + DepthViewColorScheme_DistanceMap, + DepthViewColorScheme_StarryNight, + DepthViewColorScheme_BlackAndWhite +}; + +gxf_result_t DepthView::registerInterface(gxf::Registrar* registrar) { + gxf::Expected result; + result &= registrar->parameter( + data_receiver_, "data_receiver", + "Receiver to get the data", ""); + result &= registrar->parameter( + output_tensor_, "output_tensor", + "The name of the tensor to be transmitted", ""); + result &= registrar->parameter( + output_as_tensor_, "output_as_tensor", + "The flag to indicate that output is a tensor or video buffer", "", true); + result &= registrar->parameter(data_transmitter_, "data_transmitter", + "Transmitter to send the data", ""); + result &= registrar->parameter( + allocator_, "allocator", + "Memory pool for allocating output data", ""); + result &= registrar->parameter( + color_scheme_, "color_scheme", + "The color scheme for Bi3d depth", + "Following are supported color schemes: " + "1 - ErrorColor" + "2 - RedGreenBlack" + "3 - DistanceMap" + "4 - StarryNight" + "5 - BlackAndWhite", + 5); + result &= registrar->parameter( + min_disparity_, "min_disparity", + "Minimum disparity used for colorization", "0.0f"); + result &= registrar->parameter( + max_disparity_, "max_disparity", + "Maximum disparity used for colorization", "255.0f"); + return gxf::ToResultCode(result); +} + +gxf_result_t DepthView::start() { + switch (color_scheme_) { + case DepthViewColorScheme_ErrorColor: + gradient_ = ::nvidia::isaac::ErrorColorGradient(); + break; + + case DepthViewColorScheme_RedGreenBlack: + gradient_ = ::nvidia::isaac::ColorGradient({ + ::nvidia::isaac::Pixel3ub{0, 0, 0}, ::nvidia::isaac::Pixel3ub{118, 185, 0}, + ::nvidia::isaac::Pixel3ub{221, 86, 47}, ::nvidia::isaac::Pixel3ub{255, 0, 0}}); + break; + + case DepthViewColorScheme_DistanceMap: + gradient_ = ::nvidia::isaac::DistanceColorGradient(); + break; + + case DepthViewColorScheme_StarryNight: + gradient_ = ::nvidia::isaac::StarryNightColorGradient(); + break; + + case DepthViewColorScheme_BlackAndWhite: + gradient_ = ::nvidia::isaac::BlackWhiteColorGradient(); + break; + + default: + gradient_ = ::nvidia::isaac::BlackWhiteColorGradient(); + break; + } + return GXF_SUCCESS; +} + +gxf_result_t DepthView::tick() { + auto entity = data_receiver_->receive(); + if (!entity) { + return gxf::ToResultCode(entity); + } + + // Get input + auto input_frame = entity.value().get(); + if (!input_frame) { + GXF_LOG_ERROR("Failed to get input frame from message"); + return gxf::ToResultCode(input_frame); + } + + if (input_frame.value()->video_frame_info().color_format != + gxf::VideoFormat::GXF_VIDEO_FORMAT_D32F) { + GXF_LOG_ERROR("Only supports D32F images"); + return GXF_FAILURE; + } + + if (input_frame.value()->storage_type() != gxf::MemoryStorageType::kHost) { + GXF_LOG_ERROR("Only supports video buffer on Host."); + return GXF_FAILURE; + } + + int inputWidth = input_frame.value()->video_frame_info().width; + int inputHeight = input_frame.value()->video_frame_info().height; + ::nvidia::isaac::ImageView1f input_depthview = ::nvidia::isaac::CreateImageView( + reinterpret_cast(input_frame.value()->pointer()), + inputHeight, inputWidth); + + // Create output message + auto output_message = gxf::Entity::New(context()); + if (!output_message) { + return gxf::ToResultCode(output_message); + } + + auto output_name = output_tensor_.try_get() ? output_tensor_.try_get()->c_str() : nullptr; + ::nvidia::isaac::ImageView3ub output_image; + + if (output_as_tensor_) { + // Create tensor + auto output_tensor = output_message.value().add(output_name); + if (!output_tensor) { + return gxf::ToResultCode(output_tensor); + } + + const gxf::Shape shape({inputHeight, inputWidth, 3}); + auto result = + output_tensor.value()->reshapeCustom(shape, gxf::PrimitiveType::kUnsigned8, 1, + gxf::Unexpected{GXF_UNINITIALIZED_VALUE}, + gxf::MemoryStorageType::kHost, allocator_); + if (!result) { + GXF_LOG_ERROR("reshape tensor failed."); + return GXF_FAILURE; + } + + output_image = ::nvidia::isaac::CreateImageView( + output_tensor.value()->data().value(), + inputHeight, inputWidth); + } else { + // Create video buffer + auto output_frame = output_message.value().add(output_name); + if (!output_frame) { + return gxf::ToResultCode(output_frame); + } + + std::array planes{ + gxf::ColorPlane("RGB", 3, inputWidth * 3)}; + gxf::VideoFormatSize video_format_size; + const uint64_t size = video_format_size.size(inputWidth, inputHeight, planes); + const gxf::VideoBufferInfo buffer_info{static_cast(inputWidth), + static_cast(inputHeight), + gxf::VideoFormat::GXF_VIDEO_FORMAT_RGB, + {planes.begin(), planes.end()}, + gxf::SurfaceLayout::GXF_SURFACE_LAYOUT_PITCH_LINEAR}; + output_frame.value()->resizeCustom( + buffer_info, size, + gxf::MemoryStorageType::kHost, allocator_); + output_image = ::nvidia::isaac::CreateImageView( + output_frame.value()->pointer(), + inputHeight, inputWidth); + } + + ::nvidia::isaac::Colorize(input_depthview, gradient_, min_disparity_, + max_disparity_, output_image); + + // Send the data + return gxf::ToResultCode(data_transmitter_->publish(output_message.value())); +} + +} // namespace bi3d +} // namespace isaac +} // namespace nvidia diff --git a/isaac_ros_bi3d/gxf/extensions/bi3d/components/bi3d_depthview.hpp b/isaac_ros_bi3d/gxf/extensions/bi3d/components/bi3d_depthview.hpp new file mode 100644 index 0000000..a9647aa --- /dev/null +++ b/isaac_ros_bi3d/gxf/extensions/bi3d/components/bi3d_depthview.hpp @@ -0,0 +1,63 @@ +// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES +// Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 +#pragma once + +#include +#include + +#include "engine/gems/image/color.hpp" +#include "gxf/std/codelet.hpp" +#include "gxf/std/receiver.hpp" +#include "gxf/std/tensor.hpp" +#include "gxf/std/transmitter.hpp" + +namespace nvidia { +namespace isaac { +namespace bi3d { + +class DepthView : public gxf::Codelet { + public: + gxf_result_t registerInterface(gxf::Registrar* registrar) override; + gxf_result_t start() override; + gxf_result_t tick() override; + gxf_result_t stop() override {return GXF_SUCCESS;}; + + private: + // Data receiver to get data + gxf::Parameter> data_receiver_; + // The name of the output tensor + gxf::Parameter output_tensor_; + // The flag that indicates if output as a tensor + gxf::Parameter output_as_tensor_; + // Data transmitter to send the data + gxf::Parameter> data_transmitter_; + // Allocator + gxf::Parameter> allocator_; + // Color scheme + gxf::Parameter color_scheme_; + // Minimum disparity value + gxf::Parameter min_disparity_; + // maximum disparity value + gxf::Parameter max_disparity_; + + // Color gradient used to color different depths + ::nvidia::isaac::ColorGradient gradient_; +}; + +} // namespace bi3d +} // namespace isaac +} // namespace nvidia diff --git a/isaac_ros_bi3d/gxf/extensions/bi3d/components/bi3d_inference.cpp b/isaac_ros_bi3d/gxf/extensions/bi3d/components/bi3d_inference.cpp new file mode 100644 index 0000000..278f6e2 --- /dev/null +++ b/isaac_ros_bi3d/gxf/extensions/bi3d/components/bi3d_inference.cpp @@ -0,0 +1,551 @@ +// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES +// Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +#include +#include +#include +#include + +#include "extensions/bi3d/components/bi3d_inference.hpp" + +#include "gxf/cuda/cuda_stream_id.hpp" +#include "gxf/cuda/cuda_stream_pool.hpp" +#include "gxf/multimedia/video.hpp" +#include "gxf/std/tensor.hpp" +#include "gxf/std/timestamp.hpp" + +namespace nvidia { +namespace isaac { +using TensorLayout = cvcore::tensor_ops::TensorLayout; +using ChannelType = cvcore::tensor_ops::ChannelType; +using ChannelCount = cvcore::tensor_ops::ChannelCount; + +namespace detail { + +// Function to bind a cuda stream with cid into downstream message +gxf_result_t BindCudaStream(gxf::Entity& entity, gxf_uid_t cid) { + if (cid == kNullUid) { + GXF_LOG_ERROR("stream_cid is null"); + return GXF_FAILURE; + } + auto output_stream_id = entity.add("stream"); + if (!output_stream_id) { + GXF_LOG_ERROR("failed to add cudastreamid."); + return output_stream_id.error(); + } + output_stream_id.value()->stream_cid = cid; + return GXF_SUCCESS; +} + +// Function to record a new cuda event +gxf_result_t BindCudaEvent(gxf::Entity & entity, + gxf::Handle & stream) { + cudaEvent_t event_id; + cudaEventCreateWithFlags(&event_id, 0); + gxf::CudaEvent event; + event.initWithEvent(event_id, stream->dev_id(), [](auto){}); + + auto ret = stream->record(event.event().value(), + [event = event_id, entity = entity.clone().value()](auto) { + cudaEventDestroy(event); + }); + if (!ret) { + GXF_LOG_ERROR("record event failed"); + return ret.error(); + } + + return GXF_SUCCESS; +} + +} // namespace detail + +gxf_result_t Bi3DInference::registerInterface(gxf::Registrar* registrar) { + gxf::Expected result; + + result &= registrar->parameter( + left_image_name_, "left_image_name", + "The name of the left image to be received", "", + gxf::Registrar::NoDefaultParameter(), GXF_PARAMETER_FLAGS_OPTIONAL); + result &= registrar->parameter( + right_image_name_, "right_image_name", + "The name of the right image to be received", "", + gxf::Registrar::NoDefaultParameter(), GXF_PARAMETER_FLAGS_OPTIONAL); + result &= registrar->parameter( + output_name_, "output_name", + "The name of the tensor to be passed to next node", ""); + result &= registrar->parameter( + stream_pool_, "stream_pool", + "cuda stream pool", "cuda stream pool object", + gxf::Registrar::NoDefaultParameter(), GXF_PARAMETER_FLAGS_OPTIONAL); + result &= registrar->parameter( + pool_, "pool", + "Memory pool for allocating output data", ""); + result &= registrar->parameter( + left_image_receiver_, "left_image_receiver", + "Receiver to get the left image", ""); + result &= registrar->parameter( + right_image_receiver_, "right_image_receiver", + "Receiver to get the right image", ""); + result &= registrar->parameter( + disparity_receiver_, "disparity_receiver", + "Optional receilver for dynamic disparity input", "", + gxf::Registrar::NoDefaultParameter(), GXF_PARAMETER_FLAGS_OPTIONAL); + result &= registrar->parameter( + output_transmitter_, "output_transmitter", + "Transmitter to send the data", ""); + result &= registrar->parameter( + image_type_, "image_type", + "Type of input image: BGR_U8 or RGB_U8", "", + gxf::Registrar::NoDefaultParameter(), GXF_PARAMETER_FLAGS_OPTIONAL); + result &= registrar->parameter( + pixel_mean_, "pixel_mean", + "The mean for each channel", ""); + result &= registrar->parameter( + normalization_, "normalization", + "The normalization for each channel", ""); + result &= registrar->parameter( + standard_deviation_, "standard_deviation", + "The standard deviation for each channel", ""); + result &= registrar->parameter( + max_batch_size_, "max_batch_size", + "The max batch size to run inference on", ""); + result &= registrar->parameter( + input_layer_width_, "input_layer_width", + "The model input layer width", ""); + result &= registrar->parameter( + input_layer_height_, "input_layer_height", + "The model input layer height", ""); + result &= registrar->parameter( + model_input_type_, "model_input_type", + "The model input image: BGR_U8 or RGB_U8", "", + gxf::Registrar::NoDefaultParameter(), GXF_PARAMETER_FLAGS_OPTIONAL); + result &= registrar->parameter( + featnet_engine_file_path_, "featnet_engine_file_path", + "The path to the serialized TRT engine of featnet", ""); + result &= registrar->parameter( + featnet_input_layers_name_, "featnet_input_layers_name", + "The names of the input layers", ""); + result &= registrar->parameter( + featnet_output_layers_name_, "featnet_output_layers_name", + "The names of the output layers", ""); + result &= registrar->parameter( + featnet_hr_engine_file_path_, "featnet_hr_engine_file_path", + "The path to the serialized TRT engine of high-resolution featnet", "", + gxf::Registrar::NoDefaultParameter(), GXF_PARAMETER_FLAGS_OPTIONAL); + result &= registrar->parameter( + featnet_hr_input_layers_name_, "featnet_hr_input_layers_name", + "The names of the input layers", "", + gxf::Registrar::NoDefaultParameter(), GXF_PARAMETER_FLAGS_OPTIONAL); + result &= registrar->parameter( + featnet_hr_output_layers_name_, "featnet_hr_output_layers_name", + "The names of the output layers", "", + gxf::Registrar::NoDefaultParameter(), GXF_PARAMETER_FLAGS_OPTIONAL); + result &= registrar->parameter( + refinenet_engine_file_path_, "refinenet_engine_file_path", + "The path to the serialized TRT engine of refinenet", "", + gxf::Registrar::NoDefaultParameter(), GXF_PARAMETER_FLAGS_OPTIONAL); + result &= registrar->parameter( + refinenet_input_layers_name_, "refinenet_input_layers_name", + "The names of the input layers", "", + gxf::Registrar::NoDefaultParameter(), GXF_PARAMETER_FLAGS_OPTIONAL); + result &= registrar->parameter( + refinenet_output_layers_name_, "refinenet_output_layers_name", + "The names of the output layers", "", + gxf::Registrar::NoDefaultParameter(), GXF_PARAMETER_FLAGS_OPTIONAL); + result &= registrar->parameter( + segnet_engine_file_path_, "segnet_engine_file_path", + "The path to the serialized TRT engine of segnet", ""); + result &= registrar->parameter( + segnet_input_layers_name_, "segnet_input_layers_name", + "The names of the input layers", ""); + result &= registrar->parameter( + segnet_output_layers_name_, "segnet_output_layers_name", + "The names of the output layers", ""); + result &= registrar->parameter( + engine_type_, "engine_type", + "The type of engine to be run", ""); + result &= registrar->parameter( + apply_sigmoid_, "apply_sigmoid", + "Whether to apply sigmoid operation", ""); + result &= registrar->parameter( + apply_thresholding_, "apply_thresholding", + "Whether to apply threshold operation", ""); + result &= registrar->parameter( + threshold_value_low_, "threshold_value_low", + "Low value set by thresholding operation", ""); + result &= registrar->parameter( + threshold_value_high_, "threshold_value_high", + "High value set by thresholding operation", ""); + result &= registrar->parameter( + threshold_, "threshold", + "Pixel value used by thresholding operation that casts to low or high", ""); + result &= registrar->parameter( + apply_edge_refinement_, "apply_edge_refinement", + "Whether to apply edge refinement", "", false); + result &= registrar->parameter( + max_disparity_levels_, "max_disparity_levels", + "The maximum number of output disparity levels", ""); + result &= registrar->parameter( + disparity_values_, "disparity_values", + "Input disparity values array", "", + gxf::Registrar::NoDefaultParameter(), GXF_PARAMETER_FLAGS_OPTIONAL); + result &= registrar->parameter( + timestamp_policy_, "timestamp_policy", + "Input channel to get timestamp 0(left)/1(right)", "", 0); + return gxf::ToResultCode(result); +} + +gxf_result_t Bi3DInference::start() { + // Allocate cuda stream using stream pool if necessary + if (stream_pool_.try_get()) { + auto stream = stream_pool_.try_get().value()->allocateStream(); + if (!stream) { + GXF_LOG_ERROR("allocating stream failed."); + return GXF_FAILURE; + } + cuda_stream_ = std::move(stream.value()); + if (!cuda_stream_->stream()) { + GXF_LOG_ERROR("allocated stream is not initialized."); + return GXF_FAILURE; + } + } + + // Setting image pre-processing params for Bi3D + const auto& pixel_mean_vec = pixel_mean_.get(); + const auto& normalization_vec = normalization_.get(); + const auto& standard_deviation_vec = standard_deviation_.get(); + if (pixel_mean_vec.size() != 3 || + normalization_vec.size() != 3 || + standard_deviation_vec.size() != 3) { + GXF_LOG_ERROR("Invalid preprocessing params."); + return GXF_FAILURE; + } + std::copy(pixel_mean_vec.begin(), pixel_mean_vec.end(), + preProcessorParams.pixelMean); + std::copy(normalization_vec.begin(), normalization_vec.end(), + preProcessorParams.normalization); + std::copy(standard_deviation_vec.begin(), standard_deviation_vec.end(), + preProcessorParams.stdDev); + + // Setting model input params for Bi3D + modelInputParams.maxBatchSize = max_batch_size_.get(); + modelInputParams.inputLayerWidth = input_layer_width_.get(); + modelInputParams.inputLayerHeight = input_layer_height_.get(); + + // Setting inference params for Bi3D + auto toComputeEngine = [](std::string type) -> gxf::Expected { + if (type == "GPU") { + return -1; + } else if (type == "DLA_CORE_0") { + return 0; + } else if (type == "DLA_CORE_1") { + return 1; + } else { + GXF_LOG_ERROR("Invalid compute engine type."); + return gxf::Unexpected{GXF_FAILURE}; + } + }; + auto engineType = toComputeEngine(engine_type_.get()); + if (!engineType) { + return engineType.error(); + } + featureInferenceParams = + cvcore::inferencer::TensorRTInferenceParams{ + cvcore::inferencer::TRTInferenceType::TRT_ENGINE, + nullptr, + "", + featnet_engine_file_path_.get(), + false, + cvcore::inferencer::OnnxModelBuildFlag::NONE, + 0, + 1, + featnet_input_layers_name_.get(), + featnet_output_layers_name_.get(), + engineType.value()}; + featureHRInferenceParams = + cvcore::inferencer::TensorRTInferenceParams{ + cvcore::inferencer::TRTInferenceType::TRT_ENGINE, + nullptr, + "", + featnet_hr_engine_file_path_.try_get().value_or(""), + false, + cvcore::inferencer::OnnxModelBuildFlag::NONE, + 0, + 1, + featnet_hr_input_layers_name_.try_get().value_or(std::vector{""}), + featnet_hr_output_layers_name_.try_get().value_or(std::vector{""}), + engineType.value()}; + refinementInferenceParams = + cvcore::inferencer::TensorRTInferenceParams{ + cvcore::inferencer::TRTInferenceType::TRT_ENGINE, + nullptr, + "", + refinenet_engine_file_path_.try_get().value_or(""), + false, + cvcore::inferencer::OnnxModelBuildFlag::NONE, + 0, + 1, + refinenet_input_layers_name_.try_get().value_or(std::vector{""}), + refinenet_output_layers_name_.try_get().value_or(std::vector{""}), + engineType.value()}; + segmentationInferenceParams = + cvcore::inferencer::TensorRTInferenceParams{ + cvcore::inferencer::TRTInferenceType::TRT_ENGINE, + nullptr, + "", + segnet_engine_file_path_.get(), + false, + cvcore::inferencer::OnnxModelBuildFlag::NONE, + 0, + 1, + segnet_input_layers_name_.get(), + segnet_output_layers_name_.get(), + engineType.value()}; + + // Setting extra params for Bi3D + auto toProcessingControl = [](bool flag) { + return flag ? bi3d::ProcessingControl::ENABLE : + bi3d::ProcessingControl::DISABLE; + }; + extraParams = {max_disparity_levels_.get(), + toProcessingControl(apply_edge_refinement_.get()), + toProcessingControl(apply_sigmoid_.get()), + toProcessingControl(apply_thresholding_.get()), + threshold_value_low_.get(), + threshold_value_high_.get(), + threshold_.get()}; + + // Setting Bi3D object with the provided params + objBi3D.reset(new bi3d::Bi3D( + preProcessorParams, modelInputParams, featureInferenceParams, + featureHRInferenceParams, refinementInferenceParams, + segmentationInferenceParams, extraParams)); + + const auto& disparity_values_vec = disparity_values_.try_get(); + if (disparity_values_vec) { + if (disparity_values_vec.value().empty() || + disparity_values_vec.value().size() > max_disparity_levels_.get()) { + GXF_LOG_ERROR("Invalid disparity values."); + return GXF_FAILURE; + } + disparityValues = cvcore::tensor_ops::Array(disparity_values_vec.value().size(), true); + disparityValues.setSize(disparity_values_vec.value().size()); + + for (size_t i = 0; i < disparity_values_vec.value().size(); i++) { + disparityValues[i] = disparity_values_vec.value()[i]; + } + } + return GXF_SUCCESS; +} + +gxf_result_t Bi3DInference::tick() { + // Get a CUDA stream for execution + cudaStream_t cuda_stream = 0; + if (cuda_stream_.try_get()) { + cuda_stream = cuda_stream_->stream().value(); + } + + // Receiving the data synchronously across streams + auto inputLeftMessage = left_image_receiver_->receive(); + if (!inputLeftMessage) { + return inputLeftMessage.error(); + } else if (cuda_stream != 0) { + detail::BindCudaEvent(inputLeftMessage.value(), cuda_stream_); + auto inputLeft_stream_id = inputLeftMessage.value().get("stream"); + if (inputLeft_stream_id) { + auto inputLeft_stream = gxf::Handle::Create( + inputLeft_stream_id.value().context(), + inputLeft_stream_id.value()->stream_cid); + // NOTE: This is an expensive call. It will halt the current CPU thread until all events + // previously associated with the stream are cleared + inputLeft_stream.value()->syncStream(); + } + } + auto inputRightMessage = right_image_receiver_->receive(); + if (!inputRightMessage) { + return inputRightMessage.error(); + } else if (cuda_stream != 0) { + detail::BindCudaEvent(inputRightMessage.value(), cuda_stream_); + auto inputRight_stream_id = inputRightMessage.value().get("stream"); + if (inputRight_stream_id) { + auto inputRight_stream = + gxf::Handle::Create( + inputRight_stream_id.value().context(), + inputRight_stream_id.value()->stream_cid); + // NOTE: This is an expensive call. It will halt the current CPU thread until all events + // previously associated with the stream are cleared + inputRight_stream.value()->syncStream(); + } + } + + auto maybeLeftName = left_image_name_.try_get(); + auto inputLeftBuffer = inputLeftMessage.value().get( + maybeLeftName ? maybeLeftName.value().c_str() : nullptr); + if (!inputLeftBuffer) { + return inputLeftBuffer.error(); + } + auto maybeRightName = right_image_name_.try_get(); + auto inputRightBuffer = inputRightMessage.value().get( + maybeRightName ? maybeRightName.value().c_str() : nullptr); + if (!inputRightBuffer) { + return inputRightBuffer.error(); + } + if (inputLeftBuffer.value()->storage_type() != gxf::MemoryStorageType::kDevice || + inputRightBuffer.value()->storage_type() != gxf::MemoryStorageType::kDevice) { + GXF_LOG_ERROR("input images must be on GPU."); + return GXF_FAILURE; + } + const auto& left_info = inputLeftBuffer.value()->video_frame_info(); + const auto& right_info = inputRightBuffer.value()->video_frame_info(); + if (left_info.color_format != right_info.color_format || + left_info.height != right_info.height || left_info.width != right_info.width || + left_info.surface_layout != left_info.surface_layout || + (left_info.color_format != gxf::VideoFormat::GXF_VIDEO_FORMAT_RGB && + left_info.color_format != gxf::VideoFormat::GXF_VIDEO_FORMAT_R32_G32_B32 && + left_info.color_format != gxf::VideoFormat::GXF_VIDEO_FORMAT_BGR && + left_info.color_format != gxf::VideoFormat::GXF_VIDEO_FORMAT_B32_G32_R32)) { + return GXF_INVALID_DATA_FORMAT; + } + + // Create output message + gxf::Expected outputMessage = gxf::Entity::New(context()); + if (!outputMessage) { + GXF_LOG_ERROR("Bi3D::tick ==> Failed to create output message."); + return outputMessage.error(); + } + + // Receive the disparity array if necessary + const auto& maybeDisparityReceiver = disparity_receiver_.try_get(); + if (maybeDisparityReceiver) { + auto disparityMessage = maybeDisparityReceiver.value()->receive(); + if (!disparityMessage) { + return disparityMessage.error(); + } + auto disparityTensor = disparityMessage.value().get(); + if (!disparityTensor) { + return disparityTensor.error(); + } + if (disparityTensor.value()->element_count() > max_disparity_levels_.get() || + disparityTensor.value()->element_type() != gxf::PrimitiveType::kInt32 || + disparityTensor.value()->storage_type() == gxf::MemoryStorageType::kDevice) { + GXF_LOG_ERROR("invalid input disparity values."); + return GXF_FAILURE; + } + const auto disparityCount = disparityTensor.value()->element_count(); + disparityValues = cvcore::tensor_ops::Array(disparityCount, true); + disparityValues.setSize(disparityCount); + + for (size_t i = 0; i < disparityCount; i++) { + disparityValues[i] = disparityTensor.value()->data().value()[i]; + } + + auto forwardedDisparity = outputMessage.value().add(disparityTensor->name()); + if (strcmp(disparityTensor->name(), output_name_.get().c_str()) == 0) { + GXF_LOG_ERROR("Bi3D::tick ==> Forwarded disparity array name and network" + "output name must differ."); + return GXF_FAILURE; + } + if (!forwardedDisparity) { + GXF_LOG_ERROR("Bi3D::tick ==> Failed to forward input disparity tensor in output message."); + return forwardedDisparity.error(); + } + *forwardedDisparity.value() = std::move(*disparityTensor.value()); + } + + // Creating GXF tensor to hold the data to be transmitted + auto outputImage = outputMessage.value().add(output_name_.get().c_str()); + if (!outputImage) { + GXF_LOG_ERROR("Bi3D::tick ==> Failed to create tensor in output message."); + return outputImage.error(); + } + auto result = outputImage.value()->reshape({static_cast(disparityValues.getSize()), + static_cast(left_info.height), + static_cast(left_info.width)}, + gxf::MemoryStorageType::kDevice, pool_); + if (!result) { + GXF_LOG_ERROR("Bi3D::tick ==> Failed to allocate tensor in output message."); + return result.error(); + } + + // Creating CVCore Tensors to hold the input and output data + cvcore::tensor_ops::Tensor outputImageDevice( + left_info.width, left_info.height, disparityValues.getSize(), + const_cast(outputImage.value()->data().value()), false); + + // Running the inference + if (left_info.color_format == gxf::VideoFormat::GXF_VIDEO_FORMAT_RGB || + left_info.color_format == gxf::VideoFormat::GXF_VIDEO_FORMAT_BGR) { + cvcore::tensor_ops::Tensor leftImageDevice( + left_info.width, left_info.height, + reinterpret_cast(inputLeftBuffer.value()->pointer()), false); + cvcore::tensor_ops::Tensor rightImageDevice( + right_info.width, right_info.height, + reinterpret_cast(inputRightBuffer.value()->pointer()), false); + objBi3D->execute(outputImageDevice, + leftImageDevice, rightImageDevice, + disparityValues, cuda_stream); + } else if (left_info.color_format == gxf::VideoFormat::GXF_VIDEO_FORMAT_R32_G32_B32 || + left_info.color_format == gxf::VideoFormat::GXF_VIDEO_FORMAT_B32_G32_R32) { + cvcore::tensor_ops::Tensor leftImageDevice( + left_info.width, left_info.height, + reinterpret_cast(inputLeftBuffer.value()->pointer()), false); + cvcore::tensor_ops::Tensor rightImageDevice( + right_info.width, right_info.height, + reinterpret_cast(inputRightBuffer.value()->pointer()), false); + objBi3D->execute(outputImageDevice, + leftImageDevice, rightImageDevice, + disparityValues, cuda_stream); + } else { + return GXF_FAILURE; + } + + // Allocate a cuda event that can be used to record on each tick + if (cuda_stream_.try_get()) { + detail::BindCudaStream(outputMessage.value(), cuda_stream_.cid()); + detail::BindCudaEvent(outputMessage.value(), cuda_stream_); + } + + // Pass down timestamp if necessary + auto maybeDaqTimestamp = + timestamp_policy_.get() == 0 ? inputLeftMessage.value().get() + : inputRightMessage.value().get(); + if (maybeDaqTimestamp) { + auto outputTimestamp = + outputMessage.value().add(maybeDaqTimestamp.value().name()); + if (!outputTimestamp) { + return outputTimestamp.error(); + } + *outputTimestamp.value() = *maybeDaqTimestamp.value(); + } + + // Send the data + output_transmitter_->publish(outputMessage.value()); + return GXF_SUCCESS; +} + +gxf_result_t Bi3DInference::stop() { + objBi3D.reset(nullptr); + return GXF_SUCCESS; +} + +} // namespace isaac +} // namespace nvidia diff --git a/isaac_ros_bi3d/gxf/bi3d/extensions/bi3d/Bi3D.hpp b/isaac_ros_bi3d/gxf/extensions/bi3d/components/bi3d_inference.hpp similarity index 82% rename from isaac_ros_bi3d/gxf/bi3d/extensions/bi3d/Bi3D.hpp rename to isaac_ros_bi3d/gxf/extensions/bi3d/components/bi3d_inference.hpp index 6b503a3..91d0d93 100644 --- a/isaac_ros_bi3d/gxf/bi3d/extensions/bi3d/Bi3D.hpp +++ b/isaac_ros_bi3d/gxf/extensions/bi3d/components/bi3d_inference.hpp @@ -15,15 +15,15 @@ // // SPDX-License-Identifier: Apache-2.0 -#ifndef NVIDIA_CVCORE_BI3D_HPP -#define NVIDIA_CVCORE_BI3D_HPP +#pragma once #include #include +#include +#include #include -#include - +#include "extensions/bi3d/inference/Bi3D.h" #include "gxf/cuda/cuda_stream_pool.hpp" #include "gxf/std/allocator.hpp" #include "gxf/std/codelet.hpp" @@ -31,13 +31,12 @@ #include "gxf/std/transmitter.hpp" namespace nvidia { -namespace cvcore { - +namespace isaac { // CV-Core Bi3D GXF Codelet -class Bi3D : public gxf::Codelet { -public: - Bi3D() = default; - ~Bi3D() = default; +class Bi3DInference : public gxf::Codelet { + public: + Bi3DInference() = default; + ~Bi3DInference() = default; gxf_result_t registerInterface(gxf::Registrar* registrar) override; gxf_result_t initialize() override { @@ -51,24 +50,24 @@ class Bi3D : public gxf::Codelet { gxf_result_t tick() override; gxf_result_t stop() override; -private: + private: // gxf stream handling gxf::Handle cuda_stream_ = nullptr; // cvcore image pre-processing params for Bi3D - ::cvcore::ImagePreProcessingParams preProcessorParams; + cvcore::tensor_ops::ImagePreProcessingParams preProcessorParams; // cvcore model input params for Bi3D - ::cvcore::ModelInputParams modelInputParams; + bi3d::ModelInputParams modelInputParams; // cvcore inference params for Bi3D - ::cvcore::inferencer::TensorRTInferenceParams featureInferenceParams; - ::cvcore::inferencer::TensorRTInferenceParams featureHRInferenceParams; - ::cvcore::inferencer::TensorRTInferenceParams refinementInferenceParams; - ::cvcore::inferencer::TensorRTInferenceParams segmentationInferenceParams; + cvcore::inferencer::TensorRTInferenceParams featureInferenceParams; + cvcore::inferencer::TensorRTInferenceParams featureHRInferenceParams; + cvcore::inferencer::TensorRTInferenceParams refinementInferenceParams; + cvcore::inferencer::TensorRTInferenceParams segmentationInferenceParams; // extra params for Bi3D - ::cvcore::bi3d::Bi3D::Bi3DParams extraParams; + bi3d::Bi3D::Bi3DParams extraParams; // cvcore Bi3D object - std::unique_ptr<::cvcore::bi3d::Bi3D> objBi3D; - ::cvcore::Array disparityValues; + std::unique_ptr objBi3D; + cvcore::tensor_ops::Array disparityValues; // The name of the input left image tensor gxf::Parameter left_image_name_; @@ -76,9 +75,12 @@ class Bi3D : public gxf::Codelet { gxf::Parameter right_image_name_; // The name of the output tensor gxf::Parameter output_name_; - // Data allocator to create a tensor + // Cuda Stream Pool gxf::Parameter> stream_pool_; + // Data allocator to create a tensor gxf::Parameter> pool_; + // Data allocator to forward the disparity array + gxf::Parameter> forward_pool_; // Data receiver to get left image data gxf::Parameter> left_image_receiver_; // Data receiver to get right image data @@ -132,7 +134,5 @@ class Bi3D : public gxf::Codelet { gxf::Parameter timestamp_policy_; }; -} // namespace cvcore -} // namespace nvidia - -#endif +} // namespace isaac +} // namespace nvidia diff --git a/isaac_ros_bi3d/gxf/extensions/bi3d/components/bi3d_postprocessor.cpp b/isaac_ros_bi3d/gxf/extensions/bi3d/components/bi3d_postprocessor.cpp new file mode 100644 index 0000000..07968fc --- /dev/null +++ b/isaac_ros_bi3d/gxf/extensions/bi3d/components/bi3d_postprocessor.cpp @@ -0,0 +1,128 @@ +// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES +// Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 +#include "extensions/bi3d/components/bi3d_postprocessor.hpp" + +#include + +#include "extensions/bi3d/components/bi3d_postprocessor.cu.hpp" +#include "gems/video_buffer/allocator.hpp" +#include "gxf/std/parameter_parser_std.hpp" +#include "gxf/std/timestamp.hpp" + +namespace nvidia { +namespace isaac { +namespace bi3d { + +gxf_result_t Bi3DPostprocessor::registerInterface(gxf::Registrar* registrar) noexcept { + gxf::Expected result; + result &= registrar->parameter( + bi3d_receiver_, "bi3d_receiver", "Bi3D receiver", + "Tensor from Bi3D"); + result &= registrar->parameter( + output_transmitter_, "output_transmitter", "Output transmitter", + "The collapsed output of Bi3D"); + result &= registrar->parameter( + pool_, "pool", "Pool", + "Allocator instance for output videobuffer"); + result &= registrar->parameter( + disparity_values_, "disparity_values", "Disparity values" + "Disparities values used for Bi3D inference"); + return gxf::ToResultCode(result); +} + +gxf_result_t Bi3DPostprocessor::tick() noexcept { + // Receive bi3d output + auto bi3d_message = bi3d_receiver_->receive(); + if (!bi3d_message) { + GXF_LOG_ERROR("Failed to get message"); + return gxf::ToResultCode(bi3d_message); + } + + // Get Bi3D disparity tensor + auto bi3d_disparity_tensor = bi3d_message->get("disparity"); + if (!bi3d_disparity_tensor) { + GXF_LOG_ERROR("Failed to get disparity image tensor"); + return gxf::ToResultCode(bi3d_disparity_tensor); + } + + if (!bi3d_disparity_tensor.value()->data()) { + GXF_LOG_ERROR("Failed to get pointer to bi3d disparity data"); + return gxf::ToResultCode(bi3d_disparity_tensor.value()->data()); + } + const float* bi3d_disparity_data_ptr = bi3d_disparity_tensor.value()->data().value(); + + // Get dimensions of tensor + gxf::Shape dims = bi3d_disparity_tensor.value()->shape(); + const int image_height = dims.dimension(1); + const int image_width = dims.dimension(2); + const int image_size = dims.dimension(1) * dims.dimension(2); + + // Create output message + auto out_message = gxf::Entity::New(context()); + if (!out_message) { + GXF_LOG_ERROR("Failed to allocate message"); + return gxf::ToResultCode(out_message); + } + + // Create video buffer + auto out_video_buffer = out_message->add(); + if (!out_video_buffer) { + GXF_LOG_ERROR("Failed to allocate output video buffer"); + return gxf::ToResultCode(out_video_buffer); + } + + // Allocate video buffer on device (unpadded d32f) + auto maybe_allocation = AllocateUnpaddedVideoBuffer( + out_video_buffer.value(), image_width, image_height, gxf::MemoryStorageType::kDevice, + pool_.get()); + if (!maybe_allocation) { + GXF_LOG_ERROR("Failed to allocate output video buffer's memory."); + return gxf::ToResultCode(maybe_allocation); + } + + cudaMemset(out_video_buffer.value()->pointer(), 0, + image_size * bi3d_disparity_tensor.value()->bytes_per_element()); + + for (uint32_t i = 0; i < disparity_values_.get().size(); i++) { + cuda_postprocess(bi3d_disparity_data_ptr + (i * image_size), + reinterpret_cast(out_video_buffer.value()->pointer()), + disparity_values_.get().at(i), image_height, image_width); + } + + // Add timestamp to message if it is available from input message + std::string timestamp_name{"timestamp"}; + auto maybe_bi3d_timestamp = bi3d_message->get(); + if (maybe_bi3d_timestamp) { + auto out_timestamp = out_message.value().add(timestamp_name.c_str()); + if (!out_timestamp) { + return GXF_FAILURE; + } + *out_timestamp.value() = *maybe_bi3d_timestamp.value(); + } + + // Publish message + auto result = output_transmitter_->publish(out_message.value()); + if (!result) { + return gxf::ToResultCode(result); + } + + return GXF_SUCCESS; +} + +} // namespace bi3d +} // namespace isaac +} // namespace nvidia diff --git a/isaac_ros_bi3d/gxf/extensions/bi3d/components/bi3d_postprocessor.cu b/isaac_ros_bi3d/gxf/extensions/bi3d/components/bi3d_postprocessor.cu new file mode 100644 index 0000000..3b9f5da --- /dev/null +++ b/isaac_ros_bi3d/gxf/extensions/bi3d/components/bi3d_postprocessor.cu @@ -0,0 +1,42 @@ +/* +Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. + +NVIDIA CORPORATION and its licensors retain all intellectual property +and proprietary rights in and to this software, related documentation +and any modifications thereto. Any use, reproduction, disclosure or +distribution of this software and related documentation without an express +license agreement from NVIDIA CORPORATION is strictly prohibited. +*/ +#include "extensions/bi3d/components/bi3d_postprocessor.cu.hpp" + +namespace nvidia { +namespace isaac { +namespace bi3d { + +__global__ void postprocessing_kernel(const float* input, float* output, int disparity, + int imageHeight, int imageWidth) { + const uint32_t x = blockIdx.x * blockDim.x + threadIdx.x; + const uint32_t y = blockIdx.y * blockDim.y + threadIdx.y; + const uint32_t index = y * imageWidth + x; + if (x < imageWidth && y < imageHeight) { + if (input[index] == 1.0) { + output[index] = input[index] * disparity; + } + } +} + +uint16_t ceil_div(uint16_t numerator, uint16_t denominator) { + uint32_t accumulator = numerator + denominator - 1; + return accumulator / denominator; +} + +void cuda_postprocess(const float* input, float* output, int disparity, int imageHeight, + int imageWidth) { + dim3 block(16, 16); + dim3 grid(ceil_div(imageWidth, 16), ceil_div(imageHeight, 16), 1); + postprocessing_kernel<<>>(input, output, disparity, imageHeight, imageWidth); +} + +} // namespace bi3d +} // namespace isaac +} // namespace nvidia diff --git a/isaac_ros_bi3d/gxf/bi3d_postprocessor/bi3d_postprocess/bi3d_postprocessor.cu.hpp b/isaac_ros_bi3d/gxf/extensions/bi3d/components/bi3d_postprocessor.cu.hpp similarity index 69% rename from isaac_ros_bi3d/gxf/bi3d_postprocessor/bi3d_postprocess/bi3d_postprocessor.cu.hpp rename to isaac_ros_bi3d/gxf/extensions/bi3d/components/bi3d_postprocessor.cu.hpp index 6c7de9c..749190d 100644 --- a/isaac_ros_bi3d/gxf/bi3d_postprocessor/bi3d_postprocess/bi3d_postprocessor.cu.hpp +++ b/isaac_ros_bi3d/gxf/extensions/bi3d/components/bi3d_postprocessor.cu.hpp @@ -14,8 +14,7 @@ // limitations under the License. // // SPDX-License-Identifier: Apache-2.0 -#ifndef NVIDIA_ISAAC_ROS_EXTENSIONS_BI3D_POSTPROCESSOR_CU_HPP_ -#define NVIDIA_ISAAC_ROS_EXTENSIONS_BI3D_POSTPROCESSOR_CU_HPP_ +#pragma once #include #include @@ -23,16 +22,13 @@ #include "cuda.h" #include "cuda_runtime.h" -namespace nvidia -{ -namespace isaac_ros -{ +namespace nvidia { +namespace isaac { +namespace bi3d { -void cuda_postprocess( - const float * input, float * output, int disparity, int imageHeight, - int imageWidth); +void cuda_postprocess(const float* input, float* output, int disparity, int imageHeight, + int imageWidth); -} // namespace isaac_ros +} // namespace bi3d +} // namespace isaac } // namespace nvidia - -#endif // NVIDIA_ISAAC_ROS_EXTENSIONS_BI3D_POSTPROCESSOR_CU_HPP_ diff --git a/isaac_ros_bi3d/gxf/bi3d_postprocessor/bi3d_postprocess/bi3d_postprocessor.hpp b/isaac_ros_bi3d/gxf/extensions/bi3d/components/bi3d_postprocessor.hpp similarity index 67% rename from isaac_ros_bi3d/gxf/bi3d_postprocessor/bi3d_postprocess/bi3d_postprocessor.hpp rename to isaac_ros_bi3d/gxf/extensions/bi3d/components/bi3d_postprocessor.hpp index 6fb5e28..64fee40 100644 --- a/isaac_ros_bi3d/gxf/bi3d_postprocessor/bi3d_postprocess/bi3d_postprocessor.hpp +++ b/isaac_ros_bi3d/gxf/extensions/bi3d/components/bi3d_postprocessor.hpp @@ -14,9 +14,7 @@ // limitations under the License. // // SPDX-License-Identifier: Apache-2.0 - -#ifndef NVIDIA_ISAAC_ROS_EXTENSIONS_BI3D_POSTPROCESSOR_HPP_ -#define NVIDIA_ISAAC_ROS_EXTENSIONS_BI3D_POSTPROCESSOR_HPP_ +#pragma once #include @@ -25,31 +23,26 @@ #include "gxf/std/receiver.hpp" #include "gxf/std/transmitter.hpp" - -namespace nvidia -{ -namespace isaac_ros -{ +namespace nvidia { +namespace isaac { +namespace bi3d { // GXF codelet that takes Bi3D output tensor list and collapses it into single segmentation image -class Bi3DPostprocessor : public gxf::Codelet -{ -public: - gxf_result_t registerInterface(gxf::Registrar * registrar) noexcept override; - gxf_result_t start() noexcept override; +class Bi3DPostprocessor : public gxf::Codelet { + public: + gxf_result_t registerInterface(gxf::Registrar* registrar) noexcept override; + gxf_result_t start() noexcept override { return GXF_SUCCESS; } gxf_result_t tick() noexcept override; - gxf_result_t stop() noexcept override {return GXF_SUCCESS;} + gxf_result_t stop() noexcept override { return GXF_SUCCESS; } -private: + private: gxf::Parameter> bi3d_receiver_; gxf::Parameter> output_transmitter_; + gxf::Parameter> pool_; gxf::Parameter> disparity_values_; - gxf::Parameter disparity_tensor_name_; - gxf::Parameter disparity_values_tensor_name_; }; -} // namespace isaac_ros +} // namespace bi3d +} // namespace isaac } // namespace nvidia - -#endif // NVIDIA_ISAAC_ROS_EXTENSIONS_BI3D_POSTPROCESSOR_HPP_ diff --git a/isaac_ros_bi3d/gxf/bi3d/bi3d_postprocessor/bi3d_postprocessor_utils.hpp b/isaac_ros_bi3d/gxf/extensions/bi3d/components/bi3d_postprocessor_utils.hpp similarity index 52% rename from isaac_ros_bi3d/gxf/bi3d/bi3d_postprocessor/bi3d_postprocessor_utils.hpp rename to isaac_ros_bi3d/gxf/extensions/bi3d/components/bi3d_postprocessor_utils.hpp index d6d64c5..634c23f 100644 --- a/isaac_ros_bi3d/gxf/bi3d/bi3d_postprocessor/bi3d_postprocessor_utils.hpp +++ b/isaac_ros_bi3d/gxf/extensions/bi3d/components/bi3d_postprocessor_utils.hpp @@ -14,74 +14,74 @@ // limitations under the License. // // SPDX-License-Identifier: Apache-2.0 -#ifndef NVIDIA_ISAAC_ROS_EXTENSIONS_BI3D_POSTPROCESSOR_UTILS_HPP_ -#define NVIDIA_ISAAC_ROS_EXTENSIONS_BI3D_POSTPROCESSOR_UTILS_HPP_ +#pragma once #include #include "gxf/multimedia/video.hpp" #include "gxf/std/allocator.hpp" -namespace nvidia -{ -namespace isaac_ros -{ +namespace nvidia { +namespace isaac { +namespace bi3d { -template +template inline constexpr uint8_t GetChannelSize(); -template<> -inline constexpr uint8_t GetChannelSize() -{ +template <> +inline constexpr uint8_t GetChannelSize() { return 1; } -template<> -inline constexpr uint8_t GetChannelSize() -{ +template <> +inline constexpr uint8_t GetChannelSize() { return 4; } -template -inline const char * GetColorName(); +template <> +inline constexpr uint8_t GetChannelSize() { + return 3; +} + +template +inline const char* GetColorName(); -template<> -inline const char * GetColorName() -{ +template <> +inline const char* GetColorName() { return "gray"; } -template<> -inline const char * GetColorName() -{ - return "gray"; +template <> +inline const char* GetColorName() { + return "D"; } -template -gxf::Expected AllocateVideoBuffer( - gxf::Handle video_buffer, size_t width, - size_t height, gxf::MemoryStorageType memory_storage_type, - gxf::Handle allocator) -{ +template <> +inline const char* GetColorName() { + return "RGB"; +} + +template +gxf::Expected AllocateVideoBuffer(gxf::Handle video_buffer, size_t width, + size_t height, gxf::MemoryStorageType memory_storage_type, + gxf::Handle allocator) { if (width % 2 != 0 || height % 2 != 0) { - GXF_LOG_ERROR( - "Image width and height must be even, but received width: %zu, height: %zu", - width, height); + GXF_LOG_ERROR("Image width and height must be even, but received width: %zu, height: %zu", + width, height); return gxf::Unexpected{GXF_FAILURE}; } std::array planes{ - gxf::ColorPlane(GetColorName(), GetChannelSize(), GetChannelSize() * width)}; + gxf::ColorPlane(GetColorName(), GetChannelSize(), GetChannelSize() * width)}; gxf::VideoFormatSize video_format_size; const uint64_t size = video_format_size.size(width, height, planes); const std::vector planes_filled{planes.begin(), planes.end()}; const gxf::VideoBufferInfo buffer_info{static_cast(width), - static_cast(height), T, planes_filled, - gxf::SurfaceLayout::GXF_SURFACE_LAYOUT_PITCH_LINEAR}; + static_cast(height), T, planes_filled, + gxf::SurfaceLayout::GXF_SURFACE_LAYOUT_PITCH_LINEAR}; return video_buffer->resizeCustom(buffer_info, size, memory_storage_type, allocator); } -} // namespace isaac_ros +} // namespace bi3d +} // namespace isaac } // namespace nvidia - -#endif // NVIDIA_ISAAC_ROS_EXTENSIONS_BI3D_POSTPROCESSOR_UTILS_HPP_ diff --git a/isaac_ros_bi3d/gxf/bi3d/3dv/src/Bi3D.cpp b/isaac_ros_bi3d/gxf/extensions/bi3d/inference/Bi3D.cpp similarity index 50% rename from isaac_ros_bi3d/gxf/bi3d/3dv/src/Bi3D.cpp rename to isaac_ros_bi3d/gxf/extensions/bi3d/inference/Bi3D.cpp index 096ede1..70b065f 100644 --- a/isaac_ros_bi3d/gxf/bi3d/3dv/src/Bi3D.cpp +++ b/isaac_ros_bi3d/gxf/extensions/bi3d/inference/Bi3D.cpp @@ -15,46 +15,40 @@ // // SPDX-License-Identifier: Apache-2.0 -#include - -#include "Bi3D_detail.h" +#include -#include +#include #include +#include +#include #include -#include - -#include -#ifdef NVBENCH_ENABLE -#include -#include -#endif - -#include - -#include -#include - -#include -#include - -#include -#include +#include "extensions/bi3d/inference/Bi3D.h" +#include "extensions/bi3d/inference/Bi3D_detail.hpp" +#include "extensions/tensorops/core/BBox.h" +#include "extensions/tensorops/core/BBoxUtils.h" +#include "extensions/tensorops/core/Image.h" +#include "extensions/tensorops/core/ImageUtils.h" +#include "extensions/tensorops/core/Memory.h" +#include "extensions/tensorops/core/NppUtils.h" +#include "extensions/tensorops/core/Tensor.h" +#include "extensions/tensorops/core/Traits.h" +#include "gems/dnn_inferencer/inferencer/Errors.h" +#include "gems/dnn_inferencer/inferencer/Inferencer.h" #ifndef ENGINE_DIR #define ENGINE_DIR "models" -#endif // ENGINE_DIR - -namespace cvcore { namespace bi3d { +#endif // ENGINE_DIR +namespace nvidia { +namespace isaac { +namespace bi3d { // ============================================================================= // Model Parameters // ============================================================================= -const ImagePreProcessingParams Bi3D::defaultPreProcessorParams = -{ - BGR_U8, /**< Input Image Type. */ +const ImagePreProcessingParams Bi3D::defaultPreProcessorParams = { + cvcore::tensor_ops::ImageType::BGR_U8, /**< Input Image Type. */ {1.0/127.5, 1.0/127.5, 1.0/127.5}, /**< Image Mean value per channel (OFFSET). */ {0.00392156862, 0.00392156862, @@ -62,56 +56,66 @@ const ImagePreProcessingParams Bi3D::defaultPreProcessorParams = {-127.5, -127.5, -127.5} /**< Standard deviation values per channel. */ }; -const ModelInputParams Bi3D::defaultModelInputParams = -{ +const ModelInputParams Bi3D::defaultModelInputParams = { 32, /**< Max Batch Size */ 960, /**< Width of the model input */ 576, /**< Height of the model input */ - RGB_U8 /**< Format of the model input */ + cvcore::tensor_ops::ImageType::RGB_U8 /**< Format of the model input */ }; -const Bi3D::InferencerParams Bi3D::defaultFeatureParams = -{ - inferencer::TRTInferenceType::TRT_ENGINE, +const Bi3D::InferencerParams Bi3D::defaultFeatureParams = { + TRTInferenceType::TRT_ENGINE, nullptr, + "", ENGINE_DIR "/bi3dnet_featnet.engine", + false, + cvcore::inferencer::OnnxModelBuildFlag::NONE, + 0, 1, {{"input.1"}}, {{"97"}} }; -const Bi3D::InferencerParams Bi3D::defaultHRParams = -{ - inferencer::TRTInferenceType::TRT_ENGINE, +const Bi3D::InferencerParams Bi3D::defaultHRParams = { + TRTInferenceType::TRT_ENGINE, nullptr, + "", ENGINE_DIR "/bi3dnet_featnethr.engine", + false, + cvcore::inferencer::OnnxModelBuildFlag::NONE, + 0, 1, {{"input.1"}}, {{"10"}} }; -const Bi3D::InferencerParams Bi3D::defaultRefinementParams = -{ - inferencer::TRTInferenceType::TRT_ENGINE, +const Bi3D::InferencerParams Bi3D::defaultRefinementParams = { + TRTInferenceType::TRT_ENGINE, nullptr, + "", ENGINE_DIR "/bi3dnet_refinenet.engine", + false, + cvcore::inferencer::OnnxModelBuildFlag::NONE, + 0, 1, {{"input.1"}}, {{"6"}} }; -const Bi3D::InferencerParams Bi3D::defaultSegmentationParams = -{ - inferencer::TRTInferenceType::TRT_ENGINE, +const Bi3D::InferencerParams Bi3D::defaultSegmentationParams = { + TRTInferenceType::TRT_ENGINE, nullptr, + "", ENGINE_DIR "/bi3dnet_segnet.engine", + false, + cvcore::inferencer::OnnxModelBuildFlag::NONE, + 0, 1, {{"input.1"}}, {{"278"}} }; -const Bi3D::Bi3DParams Bi3D::defaultBi3DParams = -{ +const Bi3D::Bi3DParams Bi3D::defaultBi3DParams = { 192/3, ProcessingControl::DISABLE, ProcessingControl::DISABLE, @@ -126,141 +130,118 @@ const Bi3D::Bi3DParams Bi3D::defaultBi3DParams = // ============================================================================= template -class UnaryInferencer -{ - using InferencerFactory = inferencer::InferenceBackendFactory; +class UnaryInferencer { + using InferencerFactory = cvcore::inferencer::InferenceBackendFactory; + using LayerInfo = cvcore::inferencer::LayerInfo; - public: + public: UnaryInferencer() = delete; - + UnaryInferencer(const std::string & name, const Bi3D::InferencerParams & modelParams, cudaStream_t stream = 0) : m_name{name}, m_inferencer{nullptr}, m_modelParams{modelParams}, - m_inferencerState{ErrorCode::SUCCESS} - { + m_inferencerState{cvcore::tensor_ops::ErrorCode::SUCCESS} { this->resetInferencerState(); } - ~UnaryInferencer() - { + ~UnaryInferencer() { InferencerFactory::DestroyTensorRTInferenceBackendClient(m_inferencer); } - std::error_code getInferencerState() const - { + std::error_code getInferencerState() const { return m_inferencerState; } - std::error_code resetInferencerState() - { - if(m_inferencer) - { - m_inferencerState = InferencerFactory::DestroyTensorRTInferenceBackendClient(m_inferencer); + std::error_code resetInferencerState() { + if (m_inferencer) { + m_inferencerState = + InferencerFactory::DestroyTensorRTInferenceBackendClient(m_inferencer); } - if(!m_inferencerState) - { - m_inferencerState = InferencerFactory::CreateTensorRTInferenceBackendClient(m_inferencer, - m_modelParams); + if (!m_inferencerState) { + m_inferencerState = + InferencerFactory::CreateTensorRTInferenceBackendClient(m_inferencer, + m_modelParams); } - if(!m_inferencerState) - { + if (!m_inferencerState) { m_modelMeta = m_inferencer->getModelMetaData(); } return m_inferencerState; } - inferencer::LayerInfo getInputInfo() const - { + LayerInfo getInputInfo() const { return m_modelMeta.inputLayers.at(m_modelParams.inputLayerNames[0]); } - - inferencer::LayerInfo getOutputInfo() const - { + + LayerInfo getOutputInfo() const { return m_modelMeta.outputLayers.at(m_modelParams.outputLayerNames[0]); } std::error_code execute(OutputTensorType & response, const InputTensorType & signal, - cudaStream_t stream = 0) - { -#ifdef NVBENCH_ENABLE - const std::string testName = m_name + "_batch" + - std::to_string(m_modelParams.maxBatchSize) + "_" + - std::to_string(signal.getWidth()) + "x" + - std::to_string(signal.getHeight()) + "_GPU"; - nv::bench::Timer timerFunc = nv::bench::GPU(testName.c_str(), nv::bench::Flag::DEFAULT, stream); -#endif - - if(!m_inferencerState) - { + cudaStream_t stream = 0) { + if (!m_inferencerState) { m_inferencerState = m_inferencer->setCudaStream(stream); } // Assigning the network buffers. - if(!m_inferencerState) - { + if (!m_inferencerState) { m_inferencerState = m_inferencer->setInput(dynamic_cast(signal), m_modelParams.inputLayerNames[0]); } - if(!m_inferencerState) - { + if (!m_inferencerState) { m_inferencerState = m_inferencer->setOutput(dynamic_cast(response), m_modelParams.outputLayerNames[0]); } // Run the inference - if(!m_inferencerState) - { + if (!m_inferencerState) { m_inferencerState = m_inferencer->infer(); } return m_inferencerState; } - private: - inferencer::InferenceBackendClient m_inferencer; - + private: + cvcore::inferencer::InferenceBackendClient m_inferencer; std::string m_name; Bi3D::InferencerParams m_modelParams; - inferencer::ModelMetaData m_modelMeta; + cvcore::inferencer::ModelMetaData m_modelMeta; std::error_code m_inferencerState; }; -using FeatureExtractor = UnaryInferencer; -using DisparityDetector = UnaryInferencer; -using DisparityClassifier = UnaryInferencer; +using FeatureExtractor = UnaryInferencer; +using DisparityDetector = UnaryInferencer; +using DisparityClassifier = UnaryInferencer; // ============================================================================= // Bi3D Data Exchange // ============================================================================= -struct Bi3D::Bi3DImpl -{ +struct Bi3D::Bi3DImpl { // Memory buffers // NOTE: declared mutable because the data in these buffers must always change - mutable detail::PreprocessedImage m_preprocessedLeftImageDevice; - mutable detail::PreprocessedImage m_preprocessedRightImageDevice; - mutable detail::FeatureResponse m_leftFeaturesDevice; - mutable detail::FeatureResponse m_rightFeaturesDevice; - mutable detail::FeatureResponse m_hrFeaturesDevice; - mutable detail::FeatureResponse m_leftFeaturesPadDevice; - mutable detail::FeatureResponse m_rightFeaturesPadDevice; - mutable detail::FeatureResponse m_mergedFeaturesDevice; - mutable detail::ConfidenceMap m_confidenceMapDevice; - mutable detail::ConfidenceMap m_confidenceMapResizeDevice; - mutable detail::GuidedConfidence m_fusedDisparityDevice; - mutable detail::DisparityConfidence m_disparityConfidenceDevice; + mutable PreprocessedImage m_preprocessedLeftImageDevice; + mutable PreprocessedImage m_preprocessedRightImageDevice; + mutable FeatureResponse m_leftFeaturesDevice; + mutable FeatureResponse m_rightFeaturesDevice; + mutable FeatureResponse m_hrFeaturesDevice; + mutable FeatureResponse m_leftFeaturesPadDevice; + mutable FeatureResponse m_rightFeaturesPadDevice; + mutable FeatureResponse m_mergedFeaturesDevice; + mutable ConfidenceMap m_confidenceMapDevice; + mutable ConfidenceMap m_confidenceMapResizeDevice; + mutable GuidedConfidence m_fusedDisparityDevice; + mutable DisparityConfidence m_disparityConfidenceDevice; // Exchange buffers - mutable detail::QuantizedDisparity m_quantizedDisparityDevice; - mutable detail::DisparityConfidence m_disparityConfidenceExchangeDevice; + mutable QuantizedDisparity m_quantizedDisparityDevice; + mutable DisparityConfidence m_disparityConfidenceExchangeDevice; Bi3DParams m_bi3dParams; @@ -282,10 +263,10 @@ struct Bi3D::Bi3DImpl cudaStream_t stream) : m_preprocess{new Bi3DPreProcessor{preProcessorParams, modelInputParams, stream}}, m_featureInfer{new FeatureExtractor{"FeatureNetInferencer", featureParams, stream}}, - m_segmentationInfer{new DisparityDetector{"SegmentationNetInferencer", segmentationParams, stream}}, + m_segmentationInfer{new DisparityDetector{"SegmentationNetInferencer", + segmentationParams, stream}}, m_postprocess{new Bi3DPostProcessor{modelInputParams, bi3dParams, stream}}, - m_bi3dParams{bi3dParams} - { + m_bi3dParams{bi3dParams} { // Configuring the preprocessor outputs m_preprocessedLeftImageDevice = {m_featureInfer->getInputInfo().shape[3], m_featureInfer->getInputInfo().shape[2], @@ -302,39 +283,44 @@ struct Bi3D::Bi3DImpl // The output FeatureHRNet is concatinated together with the resized // output of SegNet to form a guidance map as input into RefineNet - if(m_bi3dParams.edgeRefinement == ProcessingControl::ENABLE) - { + if (m_bi3dParams.edgeRefinement == ProcessingControl::ENABLE) { // Load featnethr and refinenet only necessary m_hrInfer.reset(new FeatureExtractor{"FeatureHRNetInferencer", hrParams, stream}); - m_refinementInfer.reset(new DisparityClassifier{"RefinementNetInferencer", refinementParams, stream}); - m_fusedDisparityDevice = {m_refinementInfer->getInputInfo().shape[3], - m_refinementInfer->getInputInfo().shape[2], - m_refinementInfer->getInputInfo().shape[1], - m_preprocessedLeftImageDevice.isCPU()}; - m_confidenceMapResizeDevice = {m_refinementInfer->getInputInfo().shape[3], - m_refinementInfer->getInputInfo().shape[2], - m_fusedDisparityDevice.getData() - + 0*m_fusedDisparityDevice.getStride(TensorDimension::CHANNEL), - m_preprocessedLeftImageDevice.isCPU()}; - m_hrFeaturesDevice = {m_hrInfer->getOutputInfo().shape[3], - m_hrInfer->getOutputInfo().shape[2], - m_hrInfer->getOutputInfo().shape[1], - m_fusedDisparityDevice.getData() - + 1*m_fusedDisparityDevice.getStride(TensorDimension::CHANNEL), - m_preprocessedLeftImageDevice.isCPU()}; + m_refinementInfer.reset(new DisparityClassifier{"RefinementNetInferencer", + refinementParams, stream}); + m_fusedDisparityDevice = { + m_refinementInfer->getInputInfo().shape[3], + m_refinementInfer->getInputInfo().shape[2], + m_refinementInfer->getInputInfo().shape[1], + m_preprocessedLeftImageDevice.isCPU()}; + m_confidenceMapResizeDevice = { + m_refinementInfer->getInputInfo().shape[3], + m_refinementInfer->getInputInfo().shape[2], + m_fusedDisparityDevice.getData() + 0*m_fusedDisparityDevice.getStride( + TensorDimension::CHANNEL), + m_preprocessedLeftImageDevice.isCPU()}; + m_hrFeaturesDevice = { + m_hrInfer->getOutputInfo().shape[3], + m_hrInfer->getOutputInfo().shape[2], + m_hrInfer->getOutputInfo().shape[1], + m_fusedDisparityDevice.getData() + + 1*m_fusedDisparityDevice.getStride(TensorDimension::CHANNEL), + m_preprocessedLeftImageDevice.isCPU()}; } // Output SegNet: Computed disparity confidence at a given disparity level - m_confidenceMapDevice = {m_segmentationInfer->getOutputInfo().shape[3], - m_segmentationInfer->getOutputInfo().shape[2], - m_preprocessedLeftImageDevice.isCPU()}; + m_confidenceMapDevice = { + m_segmentationInfer->getOutputInfo().shape[3], + m_segmentationInfer->getOutputInfo().shape[2], + m_preprocessedLeftImageDevice.isCPU()}; // Input SegNet: The concatinated left and right extracted image // features from FeatureNet - m_mergedFeaturesDevice = {m_segmentationInfer->getInputInfo().shape[3], - m_segmentationInfer->getInputInfo().shape[2], - m_segmentationInfer->getInputInfo().shape[1], - m_preprocessedLeftImageDevice.isCPU()}; + m_mergedFeaturesDevice = { + m_segmentationInfer->getInputInfo().shape[3], + m_segmentationInfer->getInputInfo().shape[2], + m_segmentationInfer->getInputInfo().shape[1], + m_preprocessedLeftImageDevice.isCPU()}; // The output extracted right features of FetureNet are padded and shifted // horizontally by the given disparity level. The left features are zero @@ -342,46 +328,50 @@ struct Bi3D::Bi3DImpl // NOTE: This model assumes the left and right image are horizontally // aligned. That is, the transformation from camera left to camera // right is strictly along the horizontal axis relative to lab frame. - m_leftFeaturesPadDevice = {m_segmentationInfer->getInputInfo().shape[3], - m_featureInfer->getOutputInfo().shape[2], - m_featureInfer->getOutputInfo().shape[1], - m_mergedFeaturesDevice.getData() - + 0*m_mergedFeaturesDevice.getStride(TensorDimension::CHANNEL), - m_preprocessedLeftImageDevice.isCPU()}; - m_rightFeaturesPadDevice = {m_segmentationInfer->getInputInfo().shape[3], - m_featureInfer->getOutputInfo().shape[2], - m_featureInfer->getOutputInfo().shape[1], - m_mergedFeaturesDevice.getData() - + m_featureInfer->getOutputInfo().shape[1] - * m_mergedFeaturesDevice.getStride(TensorDimension::CHANNEL), - m_preprocessedLeftImageDevice.isCPU()}; + m_leftFeaturesPadDevice = { + m_segmentationInfer->getInputInfo().shape[3], + m_featureInfer->getOutputInfo().shape[2], + m_featureInfer->getOutputInfo().shape[1], + m_mergedFeaturesDevice.getData() + + 0*m_mergedFeaturesDevice.getStride(TensorDimension::CHANNEL), + m_preprocessedLeftImageDevice.isCPU()}; + m_rightFeaturesPadDevice = { + m_segmentationInfer->getInputInfo().shape[3], + m_featureInfer->getOutputInfo().shape[2], + m_featureInfer->getOutputInfo().shape[1], + m_mergedFeaturesDevice.getData() + + m_featureInfer->getOutputInfo().shape[1] + * m_mergedFeaturesDevice.getStride(TensorDimension::CHANNEL), + m_preprocessedLeftImageDevice.isCPU()}; // Output FeatureNet: Feature extraction on the left and right images respectively - m_leftFeaturesDevice = {m_featureInfer->getOutputInfo().shape[3], - m_featureInfer->getOutputInfo().shape[2], - m_featureInfer->getOutputInfo().shape[1], - m_preprocessedLeftImageDevice.isCPU()}; - m_rightFeaturesDevice = {m_featureInfer->getOutputInfo().shape[3], - m_featureInfer->getOutputInfo().shape[2], - m_featureInfer->getOutputInfo().shape[1], - m_preprocessedLeftImageDevice.isCPU()}; + m_leftFeaturesDevice = { + m_featureInfer->getOutputInfo().shape[3], + m_featureInfer->getOutputInfo().shape[2], + m_featureInfer->getOutputInfo().shape[1], + m_preprocessedLeftImageDevice.isCPU()}; + m_rightFeaturesDevice = { + m_featureInfer->getOutputInfo().shape[3], + m_featureInfer->getOutputInfo().shape[2], + m_featureInfer->getOutputInfo().shape[1], + m_preprocessedLeftImageDevice.isCPU()}; } - void resizeBuffers(std::size_t width, std::size_t height) - { - if(m_quantizedDisparityDevice.getDimCount() > 0 && + void resizeBuffers(std::size_t width, std::size_t height) { + if (m_quantizedDisparityDevice.getDimCount() > 0 && m_quantizedDisparityDevice.getWidth() == width && - m_quantizedDisparityDevice.getHeight() == height) - { + m_quantizedDisparityDevice.getHeight() == height) { return; } - m_disparityConfidenceExchangeDevice = {width, height, - m_disparityConfidenceDevice.getChannelCount(), - m_preprocessedLeftImageDevice.isCPU()}; - m_quantizedDisparityDevice = {width, height, - m_disparityConfidenceDevice.getChannelCount(), - m_preprocessedLeftImageDevice.isCPU()}; + m_disparityConfidenceExchangeDevice = { + width, height, + m_disparityConfidenceDevice.getChannelCount(), + m_preprocessedLeftImageDevice.isCPU()}; + m_quantizedDisparityDevice = { + width, height, + m_disparityConfidenceDevice.getChannelCount(), + m_preprocessedLeftImageDevice.isCPU()}; } }; @@ -403,61 +393,50 @@ Bi3D::Bi3D(const ImagePreProcessingParams & preProcessorParams, Bi3D::~Bi3D() {} -void Bi3D::execute(detail::DisparityConfidence & disparityConfidence, - const detail::PreprocessedImage & leftImage, - const detail::PreprocessedImage & rightImage, - const detail::DisparityLevels & disparityLevels, - cudaStream_t stream) -{ -#ifdef NVBENCH_ENABLE - const std::string testName = "Bi3DF32_batch" + - std::to_string(1) + "_" + - std::to_string(leftImage.getWidth()) + "x" + - std::to_string(leftImage.getHeight()) + "_GPU"; - nv::bench::Timer timerFunc = nv::bench::GPU(testName.c_str(), nv::bench::Flag::DEFAULT, stream); -#endif - - if(disparityConfidence.isCPU()) - { +void Bi3D::execute(DisparityConfidence & disparityConfidence, + const PreprocessedImage & leftImage, + const PreprocessedImage & rightImage, + const DisparityLevels & disparityLevels, + cudaStream_t stream) { + if (disparityConfidence.isCPU()) { throw std::invalid_argument("cvcore::bi3d::Bi3D::execute ==> disparityConfidence " "was allocated on host (CPU) and not device (GPU)"); } - if(leftImage.isCPU() || rightImage.isCPU()) - { + if (leftImage.isCPU() || rightImage.isCPU()) { throw std::invalid_argument("cvcore::bi3d::Bi3D::execute ==> leftImage or rightImage " "was allocated on host (CPU) and not device (GPU)"); } - if(leftImage.getChannelCount() != rightImage.getChannelCount()) - { + if (leftImage.getChannelCount() != rightImage.getChannelCount()) { throw std::invalid_argument("cvcore::bi3d::Bi3D::execute ==> " "leftImage and rightImage have incompatable channel " "sizes inconsistent with model configuration parameters"); } - if(leftImage.getWidth() != rightImage.getWidth() || - leftImage.getHeight() != rightImage.getHeight()) - { + if (leftImage.getWidth() != rightImage.getWidth() || + leftImage.getHeight() != rightImage.getHeight()) { throw std::invalid_argument("cvcore::bi3d::Bi3D::execute ==> " - "leftImage and rightImage have incompatable width and height sizes inconsistent with model configuration parameters"); + "leftImage and rightImage have incompatable width and height" + "sizes inconsistent with model configuration parameters"); } // This is a boolean flag to indicate execution of the post processing step in the case that: // 1) One of the post processing types is enabled (either Sigmoid or Thresholding) // 2) The input size is greater than the output size of SegNet - bool runPostProcessing = m_pImpl->m_bi3dParams.sigmoidPostProcessing == ProcessingControl::ENABLE || - m_pImpl->m_bi3dParams.thresholdPostProcessing == ProcessingControl::ENABLE || - disparityConfidence.getDataSize() > m_pImpl->m_disparityConfidenceDevice.getDataSize(); + bool runPostProcessing = + m_pImpl->m_bi3dParams.sigmoidPostProcessing == ProcessingControl::ENABLE || + m_pImpl->m_bi3dParams.thresholdPostProcessing == ProcessingControl::ENABLE || + disparityConfidence.getDataSize() > m_pImpl->m_disparityConfidenceDevice.getDataSize(); - if(leftImage.getWidth() != m_pImpl->m_preprocessedLeftImageDevice.getWidth() || - leftImage.getHeight() != m_pImpl->m_preprocessedLeftImageDevice.getHeight()) - { + if (leftImage.getWidth() != m_pImpl->m_preprocessedLeftImageDevice.getWidth() || + leftImage.getHeight() != m_pImpl->m_preprocessedLeftImageDevice.getHeight()) { // Running the preprocessing + m_pImpl->m_preprocess->execute(m_pImpl->m_preprocessedLeftImageDevice, m_pImpl->m_preprocessedRightImageDevice, leftImage, rightImage, stream); - + // Running feature extraction m_pImpl->m_featureInfer->execute(m_pImpl->m_leftFeaturesDevice, m_pImpl->m_preprocessedLeftImageDevice, @@ -465,9 +444,7 @@ void Bi3D::execute(detail::DisparityConfidence & disparityConfidence, m_pImpl->m_featureInfer->execute(m_pImpl->m_rightFeaturesDevice, m_pImpl->m_preprocessedRightImageDevice, stream); - } - else - { + } else { // Running feature extraction m_pImpl->m_featureInfer->execute(m_pImpl->m_leftFeaturesDevice, leftImage, stream); @@ -475,40 +452,30 @@ void Bi3D::execute(detail::DisparityConfidence & disparityConfidence, rightImage, stream); } - if(m_pImpl->m_bi3dParams.edgeRefinement == ProcessingControl::ENABLE) - { + if (m_pImpl->m_bi3dParams.edgeRefinement == ProcessingControl::ENABLE) { m_pImpl->m_hrInfer->execute(m_pImpl->m_hrFeaturesDevice, m_pImpl->m_preprocessedLeftImageDevice, stream); } - + // Moving the left image extracted features to a larger buffer for maximal // disparity calculation - detail::Pad(m_pImpl->m_leftFeaturesPadDevice, m_pImpl->m_leftFeaturesDevice, + bi3d::Pad(m_pImpl->m_leftFeaturesPadDevice, m_pImpl->m_leftFeaturesDevice, 0, 0, stream); - -#ifdef NVBENCH_ENABLE - { - const std::string testName = "DisparityInferenceLoop_loop" + - std::to_string(disparityLevels.getSize()) + "_" + - std::to_string(leftImage.getWidth()) + "x" + - std::to_string(leftImage.getHeight()) + "_GPU"; - nv::bench::Timer timerFunc = nv::bench::GPU(testName.c_str(), nv::bench::Flag::DEFAULT, stream); -#endif - for(std::size_t i = 0; i < disparityLevels.getSize(); ++i) - { + + for (std::size_t i = 0; i < disparityLevels.getSize(); ++i) { // Padding the right image extracted features to the designated disparity // level - detail::Pad(m_pImpl->m_rightFeaturesPadDevice, m_pImpl->m_rightFeaturesDevice, + bi3d::Pad(m_pImpl->m_rightFeaturesPadDevice, m_pImpl->m_rightFeaturesDevice, 0, disparityLevels[i], stream); - + // Extracted features are passively concatenated into m_mergedFeaturesDevice // See: Bi3D::Bi3DImpl::Bi3DImpl() // Computing the depth confidence map on the given disparity level m_pImpl->m_segmentationInfer->execute(m_pImpl->m_confidenceMapDevice, - m_pImpl->m_mergedFeaturesDevice, - stream); + m_pImpl->m_mergedFeaturesDevice, + stream); // In the case where edge refinement is DISABLED and no postprocessing // is needed, the output buffer to Bi3D is directly used for computing @@ -517,129 +484,113 @@ void Bi3D::execute(detail::DisparityConfidence & disparityConfidence, // or the swap buffer `m_pImpl->m_disparityConfidenceDevice` which reflects // the output format of SegNet. If no postprocessing is needed // `disparityConfidence` is directly used, else the swap buffer - // `m_pImpl->m_disparityConfidenceDevice` is used to compute the + // `m_pImpl->m_disparityConfidenceDevice` is used to compute the // first pass dispairty confidence before post processing. // Cropping and resizing the disparity confidence to refinement size - if(m_pImpl->m_bi3dParams.edgeRefinement == ProcessingControl::DISABLE) - { - if(!runPostProcessing) - { + if (m_pImpl->m_bi3dParams.edgeRefinement == ProcessingControl::DISABLE) { + if (!runPostProcessing) { m_pImpl->m_confidenceMapResizeDevice = { disparityConfidence.getWidth(), disparityConfidence.getHeight(), disparityConfidence.getData() + i*disparityConfidence.getStride(TensorDimension::CHANNEL), disparityConfidence.isCPU()}; - } - else - { + } else { m_pImpl->m_confidenceMapResizeDevice = { m_pImpl->m_disparityConfidenceDevice.getWidth(), m_pImpl->m_disparityConfidenceDevice.getHeight(), m_pImpl->m_disparityConfidenceDevice.getData() - + i*m_pImpl->m_disparityConfidenceDevice.getStride(TensorDimension::CHANNEL), + + i*m_pImpl->m_disparityConfidenceDevice.getStride( + TensorDimension::CHANNEL), m_pImpl->m_disparityConfidenceDevice.isCPU()}; } } - detail::CropAndResize(m_pImpl->m_confidenceMapResizeDevice, - m_pImpl->m_confidenceMapDevice, - {0, 0, - static_cast(m_pImpl->m_confidenceMapResizeDevice.getWidth()), - static_cast(m_pImpl->m_confidenceMapResizeDevice.getHeight())}, - {0, 0, - static_cast(m_pImpl->m_confidenceMapDevice.getWidth() - m_pImpl->m_bi3dParams.maxDisparityLevels), - static_cast(m_pImpl->m_confidenceMapDevice.getHeight())}, - tensor_ops::INTERP_LINEAR, stream); - + + bi3d::CropAndResize( + m_pImpl->m_confidenceMapResizeDevice, + m_pImpl->m_confidenceMapDevice, + {0, 0, + static_cast(m_pImpl->m_confidenceMapResizeDevice.getWidth()), + static_cast(m_pImpl->m_confidenceMapResizeDevice.getHeight())}, + {0, 0, + static_cast(m_pImpl->m_confidenceMapDevice.getWidth() - + m_pImpl->m_bi3dParams.maxDisparityLevels), + static_cast(m_pImpl->m_confidenceMapDevice.getHeight())}, + cvcore::tensor_ops::InterpolationType::INTERP_LINEAR, stream); + // The confidence map and the HR features are passively concatenated into // m_fusedDisparityDevice // See: Bi3D::Bi3DImpl::Bi3DImpl() - - if(m_pImpl->m_bi3dParams.edgeRefinement == ProcessingControl::ENABLE) - { + if (m_pImpl->m_bi3dParams.edgeRefinement == ProcessingControl::ENABLE) { // Creating an overlay of the memory sector to store the confidence results // channel-wise - detail::RefinedConfidence refinedConfidenceDevice{ + RefinedConfidence refinedConfidenceDevice{ m_pImpl->m_disparityConfidenceDevice.getWidth(), m_pImpl->m_disparityConfidenceDevice.getHeight(), m_pImpl->m_disparityConfidenceDevice.getData() + i*m_pImpl->m_disparityConfidenceDevice.getStride(TensorDimension::CHANNEL), m_pImpl->m_disparityConfidenceDevice.isCPU()}; - detail::Clear(refinedConfidenceDevice, stream); - + bi3d::Clear(refinedConfidenceDevice, stream); + // Computing the refined confidence m_pImpl->m_refinementInfer->execute(refinedConfidenceDevice, - m_pImpl->m_fusedDisparityDevice, - stream); + m_pImpl->m_fusedDisparityDevice, stream); } } -#ifdef NVBENCH_ENABLE - } -#endif - + // Running the post-processing - if(runPostProcessing) - { + if (runPostProcessing) { m_pImpl->m_postprocess->execute(disparityConfidence, - m_pImpl->m_disparityConfidenceDevice, - stream); + m_pImpl->m_disparityConfidenceDevice, stream); } } -void Bi3D::execute(detail::DisparityConfidence & disparityConfidence, - const detail::InputImage & leftImage, - const detail::InputImage & rightImage, - const detail::DisparityLevels & disparityLevels, - cudaStream_t stream) -{ +void Bi3D::execute(DisparityConfidence & disparityConfidence, + const InputImage & leftImage, + const InputImage & rightImage, + const DisparityLevels & disparityLevels, + cudaStream_t stream) { // Running the preprocessing m_pImpl->m_preprocess->execute(m_pImpl->m_preprocessedLeftImageDevice, - m_pImpl->m_preprocessedRightImageDevice, - leftImage, - rightImage, - stream); + m_pImpl->m_preprocessedRightImageDevice, + leftImage, + rightImage, + stream); // Running processing this->execute(disparityConfidence, - m_pImpl->m_preprocessedLeftImageDevice, - m_pImpl->m_preprocessedRightImageDevice, - disparityLevels, stream); + m_pImpl->m_preprocessedLeftImageDevice, + m_pImpl->m_preprocessedRightImageDevice, + disparityLevels, stream); } -void Bi3D::execute(detail::QuantizedDisparity & quantizedDisparity, - const detail::InputImage & leftImage, - const detail::InputImage & rightImage, - const detail::DisparityLevels & disparityLevels, - cudaStream_t stream) -{ -#ifdef NVBENCH_ENABLE - const std::string testName = "Bi3DU8_batch" + - std::to_string(1) + "_" + - std::to_string(leftImage.getWidth()) + "x" + - std::to_string(leftImage.getHeight()) + "_GPU"; - nv::bench::Timer timerFunc = nv::bench::GPU(testName.c_str(), nv::bench::Flag::DEFAULT, stream); -#endif - +void Bi3D::execute(QuantizedDisparity & quantizedDisparity, + const InputImage & leftImage, + const InputImage & rightImage, + const DisparityLevels & disparityLevels, + cudaStream_t stream) { // Ensuring the buffers are appropreately allocated m_pImpl->resizeBuffers(leftImage.getWidth(), leftImage.getHeight()); // Calling the floating point version of this method this->execute(m_pImpl->m_disparityConfidenceExchangeDevice, - leftImage, rightImage, disparityLevels, - stream); - + leftImage, rightImage, disparityLevels, + stream); + // Converting the confidence to U8 - detail::ConvertBitDepth(m_pImpl->m_quantizedDisparityDevice, - m_pImpl->m_disparityConfidenceExchangeDevice, - 255.0, stream); + bi3d::ConvertBitDepth(m_pImpl->m_quantizedDisparityDevice, + m_pImpl->m_disparityConfidenceExchangeDevice, + 255.0, stream); // Copying the results to output - detail::QuantizedDisparity partial{quantizedDisparity.getWidth(), - quantizedDisparity.getHeight(), - quantizedDisparity.getChannelCount(), - m_pImpl->m_quantizedDisparityDevice.getData(), - m_pImpl->m_quantizedDisparityDevice.isCPU()}; + QuantizedDisparity partial{quantizedDisparity.getWidth(), + quantizedDisparity.getHeight(), + quantizedDisparity.getChannelCount(), + m_pImpl->m_quantizedDisparityDevice.getData(), + m_pImpl->m_quantizedDisparityDevice.isCPU()}; Copy(quantizedDisparity, partial, stream); } -}} // namespace cvcore::bi3d +} // namespace bi3d +} // namespace isaac +} // namespace nvidia diff --git a/isaac_ros_bi3d/gxf/bi3d/3dv/include/cv/bi3d/Bi3D.h b/isaac_ros_bi3d/gxf/extensions/bi3d/inference/Bi3D.h similarity index 58% rename from isaac_ros_bi3d/gxf/bi3d/3dv/include/cv/bi3d/Bi3D.h rename to isaac_ros_bi3d/gxf/extensions/bi3d/inference/Bi3D.h index 81671a9..b66914a 100644 --- a/isaac_ros_bi3d/gxf/bi3d/3dv/include/cv/bi3d/Bi3D.h +++ b/isaac_ros_bi3d/gxf/extensions/bi3d/inference/Bi3D.h @@ -1,5 +1,5 @@ // SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,24 +15,37 @@ // // SPDX-License-Identifier: Apache-2.0 -#ifndef CVCORE_BI3D_H_ -#define CVCORE_BI3D_H_ - -#include +#pragma once #include +#include -#include -#include -#include -#include - -#include - -namespace cvcore { namespace bi3d { - -enum class ProcessingControl : std::uint8_t -{ +#include "extensions/bi3d/inference/Model.h" +#include "extensions/tensorops/core/Array.h" +#include "extensions/tensorops/core/ComputeEngine.h" +#include "extensions/tensorops/core/Tensor.h" +#include "gems/dnn_inferencer/inferencer/Inferencer.h" + +namespace nvidia { +namespace isaac { +namespace bi3d { + +using TensorLayout = cvcore::tensor_ops::TensorLayout; +using ChannelType = cvcore::tensor_ops::ChannelType; +using ChannelCount = cvcore::tensor_ops::ChannelCount; +using ImagePreProcessingParams = cvcore::tensor_ops::ImagePreProcessingParams; +using Tensor_HWC_C3_U8 = cvcore::tensor_ops::Tensor; +using Tensor_CHW_C3_F32 = cvcore::tensor_ops::Tensor; +using Tensor_CHW_CX_U8 = cvcore::tensor_ops::Tensor; +using Tensor_CHW_CX_F32 = cvcore::tensor_ops::Tensor; +using DisparityLevels = cvcore::tensor_ops::Array; +using TensorBase = cvcore::tensor_ops::TensorBase; + +enum class ProcessingControl : std::uint8_t { DISABLE = 0, ENABLE = 1 }; @@ -40,10 +53,10 @@ enum class ProcessingControl : std::uint8_t /** * Interface for Loading and running inference on bi3d network. */ -class Bi3D -{ - public: - using InferencerParams = inferencer::TensorRTInferenceParams; +class Bi3D { + public: + using InferencerParams = cvcore::inferencer::TensorRTInferenceParams; + using TRTInferenceType = cvcore::inferencer::TRTInferenceType; /** * Default Image Processing Params for Bi3D. @@ -62,20 +75,26 @@ class Bi3D static const InferencerParams defaultHRParams; static const InferencerParams defaultRefinementParams; static const InferencerParams defaultSegmentationParams; - + /** * Bi3D extra params */ - struct Bi3DParams - { - std::size_t maxDisparityLevels; // Maximum pixel-wise shift between left and right images - ProcessingControl edgeRefinement; // Switch to turn on/off edge refinement models (FeatNetHR and - // RefineNet) in inferencing - ProcessingControl sigmoidPostProcessing; // Switch to turn on/off sigmoid operation in postprocessing - ProcessingControl thresholdPostProcessing; // Switch to turn on/off thresholding in postprocessing - float thresholdValueLow; // The low value set by threshold postprocessing - float thresholdValueHigh; // The high value set by threshold postprocessing - float threshold; // The threshold value that casts pixel values to either low or high + struct Bi3DParams { + // Maximum pixel-wise shift between left and right images + std::size_t maxDisparityLevels; + // Switch to turn on/off edge refinement models (FeatNetHR and RefineNet) + // in inferencing + ProcessingControl edgeRefinement; + // Switch to turn on/off sigmoid operation in postprocessing + ProcessingControl sigmoidPostProcessing; + // Switch to turn on/off thresholding in postprocessing + ProcessingControl thresholdPostProcessing; + // The low value set by threshold postprocessing + float thresholdValueLow; + // The high value set by threshold postprocessing + float thresholdValueHigh; + // The threshold value that casts pixel values to either low or high + float threshold; }; static const Bi3DParams defaultBi3DParams; @@ -109,10 +128,10 @@ class Bi3D * @param rightImage right image tensor in RGB HWC format. * @param disparityLevels diparity value to run depth estimation based on. */ - void execute(Tensor & quantizedDisparity, - const Tensor & leftImage, - const Tensor & rightImage, - const Array & disparityLevels, + void execute(Tensor_CHW_CX_U8 & quantizedDisparity, + const Tensor_HWC_C3_U8 & leftImage, + const Tensor_HWC_C3_U8 & rightImage, + const DisparityLevels& disparityLevels, cudaStream_t stream = 0); /** @@ -122,10 +141,10 @@ class Bi3D * @param rightImage right image tensor in RGB HWC format. * @param disparityLevels diparity value to run depth estimation based on. */ - void execute(Tensor & disparityConfidence, - const Tensor & leftImage, - const Tensor & rightImage, - const Array & disparityLevels, + void execute(Tensor_CHW_CX_F32 & disparityConfidence, + const Tensor_HWC_C3_U8 & leftImage, + const Tensor_HWC_C3_U8 & rightImage, + const DisparityLevels& disparityLevels, cudaStream_t stream = 0); /** @@ -135,13 +154,13 @@ class Bi3D * @param rightImage right image tensor in RGB CHW format. * @param disparityLevels diparity value to run depth estimation based on. */ - void execute(Tensor & disparityConfidence, - const Tensor & leftImage, - const Tensor & rightImage, - const Array & disparityLevels, + void execute(Tensor_CHW_CX_F32 & disparityConfidence, + const Tensor_CHW_C3_F32 & leftImage, + const Tensor_CHW_C3_F32 & rightImage, + const DisparityLevels& disparityLevels, cudaStream_t stream = 0); - private: + private: struct Bi3DImpl; std::unique_ptr m_pImpl; @@ -150,9 +169,8 @@ class Bi3D /** * Interface for running pre-processing for Bi3D. */ -class Bi3DPreProcessor -{ - public: +class Bi3DPreProcessor { + public: /** * Removing the default constructor for Bi3DPreProcessor. */ @@ -163,7 +181,7 @@ class Bi3DPreProcessor * @param preProcessorParams Image preprocessing parameters. * @param modelInputParams Model input parameters. */ - Bi3DPreProcessor(const ImagePreProcessingParams & preProcessorParams, + Bi3DPreProcessor(const ImagePreProcessingParams & preProcessorParams, const ModelInputParams & modelInputParams, cudaStream_t stream = 0); @@ -172,19 +190,19 @@ class Bi3DPreProcessor */ ~Bi3DPreProcessor(); - void execute(Tensor & preprocessedLeftImage, - Tensor & preprocessedRightImage, - const Tensor & leftImage, - const Tensor & rightImage, + void execute(Tensor_CHW_C3_F32 & preprocessedLeftImage, + Tensor_CHW_C3_F32 & preprocessedRightImage, + const Tensor_HWC_C3_U8 & leftImage, + const Tensor_HWC_C3_U8 & rightImage, cudaStream_t stream = 0); - void execute(Tensor & preprocessedLeftImage, - Tensor & preprocessedRightImage, - const Tensor & leftImage, - const Tensor & rightImage, + void execute(Tensor_CHW_C3_F32 & preprocessedLeftImage, + Tensor_CHW_C3_F32 & preprocessedRightImage, + const Tensor_CHW_C3_F32 & leftImage, + const Tensor_CHW_C3_F32 & rightImage, cudaStream_t stream = 0); - private: + private: struct Bi3DPreProcessorImpl; std::unique_ptr m_pImpl; @@ -193,9 +211,8 @@ class Bi3DPreProcessor /** * Interface for running post-processing for Bi3D. */ -class Bi3DPostProcessor -{ - public: +class Bi3DPostProcessor { + public: /** * Removing the default constructor for Bi3DPostProcessor. */ @@ -215,16 +232,16 @@ class Bi3DPostProcessor */ ~Bi3DPostProcessor(); - void execute(Tensor & disparityConfidence, - Tensor & rawDisparityConfidence, + void execute(Tensor_CHW_CX_F32 & disparityConfidence, + Tensor_CHW_CX_F32 & rawDisparityConfidence, cudaStream_t stream = 0); - private: + private: struct Bi3DPostProcessorImpl; std::unique_ptr m_pImpl; }; -}} // namespace cvcore::bi3d - -#endif // CVCORE_BI3D_H_ +} // namespace bi3d +} // namespace isaac +} // namespace nvidia diff --git a/isaac_ros_bi3d/gxf/bi3d/3dv/src/Bi3DPostProcessor.cpp b/isaac_ros_bi3d/gxf/extensions/bi3d/inference/Bi3DPostProcessor.cpp similarity index 59% rename from isaac_ros_bi3d/gxf/bi3d/3dv/src/Bi3DPostProcessor.cpp rename to isaac_ros_bi3d/gxf/extensions/bi3d/inference/Bi3DPostProcessor.cpp index 2ece996..da4aebe 100644 --- a/isaac_ros_bi3d/gxf/bi3d/3dv/src/Bi3DPostProcessor.cpp +++ b/isaac_ros_bi3d/gxf/extensions/bi3d/inference/Bi3DPostProcessor.cpp @@ -15,26 +15,21 @@ // // SPDX-License-Identifier: Apache-2.0 -#include - -#include "Bi3D_detail.h" - #include -#ifdef NVBENCH_ENABLE -#include -#include -#endif - -#include +#include +#include "extensions/bi3d/inference/Bi3D.h" +#include "extensions/bi3d/inference/Bi3D_detail.hpp" +#include "extensions/tensorops/core/Memory.h" -namespace cvcore { namespace bi3d { +namespace nvidia { +namespace isaac { +namespace bi3d { -struct Bi3DPostProcessor::Bi3DPostProcessorImpl -{ - mutable detail::DisparityConfidence m_disparityConfidenceDevice; - mutable detail::DisparityConfidence m_disparityConfidenceThresholdDevice; - mutable detail::DisparityConfidence m_disparityConfidenceResizeDevice; +struct Bi3DPostProcessor::Bi3DPostProcessorImpl { + mutable DisparityConfidence m_disparityConfidenceDevice; + mutable DisparityConfidence m_disparityConfidenceThresholdDevice; + mutable DisparityConfidence m_disparityConfidenceResizeDevice; ModelInputParams m_modelParams; Bi3D::Bi3DParams m_params; @@ -55,12 +50,10 @@ struct Bi3DPostProcessor::Bi3DPostProcessorImpl false}; } - void resizeBuffers(std::size_t width, std::size_t height) - { - if(m_disparityConfidenceResizeDevice.getDimCount() > 0 && + void resizeBuffers(std::size_t width, std::size_t height) { + if (m_disparityConfidenceResizeDevice.getDimCount() > 0 && m_disparityConfidenceResizeDevice.getWidth() == width && - m_disparityConfidenceResizeDevice.getHeight() == height) - { + m_disparityConfidenceResizeDevice.getHeight() == height) { return; } @@ -81,80 +74,54 @@ Bi3DPostProcessor::Bi3DPostProcessor(const ModelInputParams & modelInputParams, Bi3DPostProcessor::~Bi3DPostProcessor() {} -void Bi3DPostProcessor::execute(detail::DisparityConfidence & disparityConfidence, - detail::DisparityConfidence & rawDisparityConfidence, - cudaStream_t stream) -{ -#ifdef NVBENCH_ENABLE - const std::string testName = "Bi3DPostProcessor_batch" + - std::to_string(m_pImpl->m_modelParams.maxBatchSize) + "_GPU"; - nv::bench::Timer timerFunc = nv::bench::GPU(testName.c_str(), nv::bench::Flag::DEFAULT, stream); -#endif - +void Bi3DPostProcessor::execute(DisparityConfidence & disparityConfidence, + DisparityConfidence & rawDisparityConfidence, + cudaStream_t stream) { // Ensuring the buffers are appropreately allocated m_pImpl->resizeBuffers(disparityConfidence.getWidth(), disparityConfidence.getHeight()); - -#ifdef NVBENCH_ENABLE - { - const std::string testName = "Bi3DPostProcessorTransformBlock_batch" + - std::to_string(m_pImpl->m_modelParams.maxBatchSize) + "_GPU"; - nv::bench::Timer timerFunc = nv::bench::GPU(testName.c_str(), nv::bench::Flag::DEFAULT, stream); -#endif - if(m_pImpl->m_params.sigmoidPostProcessing == ProcessingControl::ENABLE) - { - detail::Sigmoid(m_pImpl->m_disparityConfidenceDevice, + if (m_pImpl->m_params.sigmoidPostProcessing == ProcessingControl::ENABLE) { + Sigmoid(m_pImpl->m_disparityConfidenceDevice, rawDisparityConfidence, stream); - if(m_pImpl->m_params.thresholdPostProcessing == ProcessingControl::ENABLE) - { - detail::Threshold(m_pImpl->m_disparityConfidenceThresholdDevice, + if (m_pImpl->m_params.thresholdPostProcessing == ProcessingControl::ENABLE) { + Threshold(m_pImpl->m_disparityConfidenceThresholdDevice, m_pImpl->m_disparityConfidenceDevice, m_pImpl->m_params.thresholdValueLow, m_pImpl->m_params.thresholdValueHigh, m_pImpl->m_params.threshold, stream); - - detail::Resize(m_pImpl->m_disparityConfidenceResizeDevice, + + Resize(m_pImpl->m_disparityConfidenceResizeDevice, m_pImpl->m_disparityConfidenceThresholdDevice, - tensor_ops::INTERP_LINEAR, stream); - } - else - { - detail::Resize(m_pImpl->m_disparityConfidenceResizeDevice, + cvcore::tensor_ops::InterpolationType::INTERP_LINEAR, stream); + } else { + Resize(m_pImpl->m_disparityConfidenceResizeDevice, m_pImpl->m_disparityConfidenceDevice, - tensor_ops::INTERP_LINEAR, stream); + cvcore::tensor_ops::InterpolationType::INTERP_LINEAR, stream); } - } - else - { - if(m_pImpl->m_params.thresholdPostProcessing == ProcessingControl::ENABLE) - { - detail::Threshold(m_pImpl->m_disparityConfidenceThresholdDevice, + } else { + if (m_pImpl->m_params.thresholdPostProcessing == ProcessingControl::ENABLE) { + Threshold(m_pImpl->m_disparityConfidenceThresholdDevice, rawDisparityConfidence, m_pImpl->m_params.thresholdValueLow, m_pImpl->m_params.thresholdValueHigh, m_pImpl->m_params.threshold, stream); - - detail::Resize(m_pImpl->m_disparityConfidenceResizeDevice, + + Resize(m_pImpl->m_disparityConfidenceResizeDevice, m_pImpl->m_disparityConfidenceThresholdDevice, - tensor_ops::INTERP_LINEAR, stream); - } - else - { - detail::Resize(m_pImpl->m_disparityConfidenceResizeDevice, + cvcore::tensor_ops::InterpolationType::INTERP_LINEAR, stream); + } else { + Resize(m_pImpl->m_disparityConfidenceResizeDevice, rawDisparityConfidence, - tensor_ops::INTERP_LINEAR, stream); + cvcore::tensor_ops::InterpolationType::INTERP_LINEAR, stream); } } -#ifdef NVBENCH_ENABLE - } -#endif - - detail::DisparityConfidence partial{disparityConfidence.getWidth(), + DisparityConfidence partial{disparityConfidence.getWidth(), disparityConfidence.getHeight(), disparityConfidence.getChannelCount(), m_pImpl->m_disparityConfidenceResizeDevice.getData(), m_pImpl->m_disparityConfidenceResizeDevice.isCPU()}; Copy(disparityConfidence, partial, stream); } - -}} // namespace cvcore::bi3d +} // namespace bi3d +} // namespace isaac +} // namespace nvidia diff --git a/isaac_ros_bi3d/gxf/bi3d/3dv/src/Bi3DPreProcessor.cpp b/isaac_ros_bi3d/gxf/extensions/bi3d/inference/Bi3DPreProcessor.cpp similarity index 54% rename from isaac_ros_bi3d/gxf/bi3d/3dv/src/Bi3DPreProcessor.cpp rename to isaac_ros_bi3d/gxf/extensions/bi3d/inference/Bi3DPreProcessor.cpp index ddbb2d9..01b351e 100644 --- a/isaac_ros_bi3d/gxf/bi3d/3dv/src/Bi3DPreProcessor.cpp +++ b/isaac_ros_bi3d/gxf/extensions/bi3d/inference/Bi3DPreProcessor.cpp @@ -15,44 +15,40 @@ // // SPDX-License-Identifier: Apache-2.0 -#include - -#include "Bi3D_detail.h" - #include -#ifdef NVBENCH_ENABLE -#include -#include -#endif - -#include +#include +#include "extensions/bi3d/inference/Bi3D.h" +#include "extensions/bi3d/inference/Bi3D_detail.hpp" +#include "extensions/bi3d/inference/Model.h" +#include "extensions/tensorops/core/Memory.h" -namespace cvcore { namespace bi3d { +namespace nvidia { +namespace isaac { +namespace bi3d { -struct Bi3DPreProcessor::Bi3DPreProcessorImpl -{ - mutable detail::InputImage m_leftImageDevice; - mutable detail::InputImage m_rightImageDevice; - mutable detail::PreprocessedImage m_preLeftImageDevice; - mutable detail::PreprocessedImage m_preRightImageDevice; - mutable detail::InputImage m_leftImageResizeDevice; - mutable detail::InputImage m_rightImageResizeDevice; +struct Bi3DPreProcessor::Bi3DPreProcessorImpl { + mutable InputImage m_leftImageDevice; + mutable InputImage m_rightImageDevice; + mutable PreprocessedImage m_preLeftImageDevice; + mutable PreprocessedImage m_preRightImageDevice; + mutable InputImage m_leftImageResizeDevice; + mutable InputImage m_rightImageResizeDevice; // Operation(s) - detail::NormalizeFunctor m_normalizer; + NormalizeFunctor m_normalizer; // Needed parameters for pre-processing ImagePreProcessingParams m_preProcessorParams; ModelInputParams m_modelParams; - Bi3DPreProcessorImpl(const ImagePreProcessingParams & preProcessorParams, + Bi3DPreProcessorImpl(const ImagePreProcessingParams & preProcessorParams, const ModelInputParams & modelInputParams, cudaStream_t stream) : m_preProcessorParams{preProcessorParams}, m_modelParams{modelInputParams}, - m_normalizer{modelInputParams.inputLayerWidth, modelInputParams.inputLayerHeight} - { + m_normalizer{static_cast(modelInputParams.inputLayerWidth), + static_cast(modelInputParams.inputLayerHeight)} { // Configuring the top-level model input(s) m_leftImageResizeDevice = {modelInputParams.inputLayerWidth, modelInputParams.inputLayerHeight, @@ -62,12 +58,10 @@ struct Bi3DPreProcessor::Bi3DPreProcessorImpl m_leftImageResizeDevice.isCPU()}; } - void resizeBuffers(std::size_t width, std::size_t height) - { - if(m_leftImageDevice.getDimCount() > 0 && + void resizeBuffers(std::size_t width, std::size_t height) { + if (m_leftImageDevice.getDimCount() > 0 && m_leftImageDevice.getWidth() == width && - m_leftImageDevice.getHeight() == height) - { + m_leftImageDevice.getHeight() == height) { return; } @@ -82,27 +76,18 @@ struct Bi3DPreProcessor::Bi3DPreProcessorImpl // Bi3D Frontend // ============================================================================= -Bi3DPreProcessor::Bi3DPreProcessor(const ImagePreProcessingParams & preProcessorParams, +Bi3DPreProcessor::Bi3DPreProcessor(const ImagePreProcessingParams & preProcessorParams, const ModelInputParams & modelInputParams, cudaStream_t stream) : m_pImpl{new Bi3DPreProcessorImpl(preProcessorParams, modelInputParams, stream)} {} Bi3DPreProcessor::~Bi3DPreProcessor() {} -void Bi3DPreProcessor::execute(detail::PreprocessedImage & preprocessedLeftImage, - detail::PreprocessedImage & preprocessedRightImage, - const detail::InputImage & leftImage, - const detail::InputImage & rightImage, - cudaStream_t stream) -{ -#ifdef NVBENCH_ENABLE - const std::string testName = "Bi3DPreProcessor_batch" + - std::to_string(m_pImpl->m_modelParams.maxBatchSize) + "_" + - std::to_string(leftImage.getWidth()) + "x" + - std::to_string(leftImage.getHeight()) + "_GPU"; - nv::bench::Timer timerFunc = nv::bench::GPU(testName.c_str(), nv::bench::Flag::DEFAULT, stream); -#endif - +void Bi3DPreProcessor::execute(PreprocessedImage & preprocessedLeftImage, + PreprocessedImage & preprocessedRightImage, + const InputImage & leftImage, + const InputImage & rightImage, + cudaStream_t stream) { // Ensuring the buffers are appropreately allocated m_pImpl->resizeBuffers(leftImage.getWidth(), leftImage.getHeight()); @@ -112,10 +97,10 @@ void Bi3DPreProcessor::execute(detail::PreprocessedImage & preprocessedLeftImage Copy(m_pImpl->m_rightImageDevice, rightImage, stream); // Resizing the data to model input size - tensor_ops::Resize(m_pImpl->m_leftImageResizeDevice, m_pImpl->m_leftImageDevice, - false, tensor_ops::INTERP_LINEAR, stream); - tensor_ops::Resize(m_pImpl->m_rightImageResizeDevice, m_pImpl->m_rightImageDevice, - false, tensor_ops::INTERP_LINEAR, stream); + Resize(m_pImpl->m_leftImageResizeDevice, m_pImpl->m_leftImageDevice, + false, cvcore::tensor_ops::InterpolationType::INTERP_LINEAR, stream); + Resize(m_pImpl->m_rightImageResizeDevice, m_pImpl->m_rightImageDevice, + false, cvcore::tensor_ops::InterpolationType::INTERP_LINEAR, stream); // Normalize (data whiten) the images m_pImpl->m_normalizer(preprocessedLeftImage, m_pImpl->m_leftImageResizeDevice, @@ -128,20 +113,11 @@ void Bi3DPreProcessor::execute(detail::PreprocessedImage & preprocessedLeftImage stream); } -void Bi3DPreProcessor::execute(detail::PreprocessedImage & preprocessedLeftImage, - detail::PreprocessedImage & preprocessedRightImage, - const detail::PreprocessedImage & leftImage, - const detail::PreprocessedImage & rightImage, - cudaStream_t stream) -{ -#ifdef NVBENCH_ENABLE - const std::string testName = "Bi3DPreProcessor_batch" + - std::to_string(m_pImpl->m_modelParams.maxBatchSize) + "_" + - std::to_string(leftImage.getWidth()) + "x" + - std::to_string(leftImage.getHeight()) + "_GPU"; - nv::bench::Timer timerFunc = nv::bench::GPU(testName.c_str(), nv::bench::Flag::DEFAULT, stream); -#endif - +void Bi3DPreProcessor::execute(PreprocessedImage & preprocessedLeftImage, + PreprocessedImage & preprocessedRightImage, + const PreprocessedImage & leftImage, + const PreprocessedImage & rightImage, + cudaStream_t stream) { // Ensuring the buffers are appropreately allocated m_pImpl->resizeBuffers(leftImage.getWidth(), leftImage.getHeight()); @@ -150,10 +126,11 @@ void Bi3DPreProcessor::execute(detail::PreprocessedImage & preprocessedLeftImage Copy(m_pImpl->m_preRightImageDevice, rightImage, stream); // Resizing the data to model input size - detail::Resize(preprocessedLeftImage, m_pImpl->m_preLeftImageDevice, - tensor_ops::INTERP_LINEAR, stream); - detail::Resize(preprocessedLeftImage, m_pImpl->m_preRightImageDevice, - tensor_ops::INTERP_LINEAR, stream); + Resize(preprocessedLeftImage, m_pImpl->m_preLeftImageDevice, + cvcore::tensor_ops::InterpolationType::INTERP_LINEAR, stream); + Resize(preprocessedLeftImage, m_pImpl->m_preRightImageDevice, + cvcore::tensor_ops::InterpolationType::INTERP_LINEAR, stream); } - -}} // namespace cvcore::bi3d \ No newline at end of file +} // namespace bi3d +} // namespace isaac +} // namespace nvidia diff --git a/isaac_ros_bi3d/gxf/bi3d/3dv/src/Bi3D_detail.cpp b/isaac_ros_bi3d/gxf/extensions/bi3d/inference/Bi3D_detail.cpp similarity index 79% rename from isaac_ros_bi3d/gxf/bi3d/3dv/src/Bi3D_detail.cpp rename to isaac_ros_bi3d/gxf/extensions/bi3d/inference/Bi3D_detail.cpp index 602c50e..2e2c50d 100644 --- a/isaac_ros_bi3d/gxf/bi3d/3dv/src/Bi3D_detail.cpp +++ b/isaac_ros_bi3d/gxf/extensions/bi3d/inference/Bi3D_detail.cpp @@ -15,20 +15,21 @@ // // SPDX-License-Identifier: Apache-2.0 -#include "Bi3D_detail.h" +#include "extensions/bi3d/inference/Bi3D_detail.hpp" -#include -#include - -#include #include +#include #include +#include +#include +#include -namespace cvcore { namespace bi3d { namespace detail { +namespace nvidia { +namespace isaac { +namespace bi3d { -void _pad(Tensor & dst, const Tensor & src, - int topBorderHeight, int leftBorderWidth, cudaStream_t stream) -{ +void _pad(Tensor_CHW_C1_F32& dst, const Tensor_CHW_C1_F32& src, + int topBorderHeight, int leftBorderWidth, cudaStream_t stream) { // Using NPP function: ``nppiCopyConstBorder_32f_C1R_Ctx'' // Spec ==> NppStatus nppiCopyConstBorder_32f_C1R_Ctx(const Npp32f * pSrc, // int nSrcStep, @@ -47,18 +48,19 @@ void _pad(Tensor & dst, const Tensor & src, // oSrcSizeROI Size (width, height) of the source region in pixels. // pDst Destination-Image Pointer. // nDstStep Destination-Image Stride in bytes. - // oDstSizeROI Size (width, height) of the destination region, i.e. the region that gets - // filled with data from the source image (inner part) and constant border - // color (outer part). - // nTopBorderHeight Height (in pixels) of the top border. The number of pixel rows at the top - // of the destination ROI that will be filled with the constant border - // color. nBottomBorderHeight = oDstSizeROI.height - nTopBorderHeight - - // oSrcSizeROI.height. - // nLeftBorderWidth Width (in pixels) of the left border. The width of the border at the - // right side of the destination ROI is implicitly defined by the size of - // the source ROI: nRightBorderWidth = oDstSizeROI.width - nLeftBorderWidth - // - oSrcSizeROI.width. - // nValue The pixel value to be set for border pixels for single channel functions. + // oDstSizeROI Size (width, height) of the destination region, i.e. the region + // that gets filled with data from the source image (inner part) + // and constant border color (outer part). + // nTopBorderHeight Height (in pixels) of the top border. The number of pixel rows at + // the top of the destination ROI that will be filled with the + // constant border color. nBottomBorderHeight = oDstSizeROI.height - + // nTopBorderHeight - oSrcSizeROI.height. + // nLeftBorderWidth Width (in pixels) of the left border. The width of the border at + // the right side of the destination ROI is implicitly defined by the + // size of the source ROI: nRightBorderWidth = oDstSizeROI.width - + // nLeftBorderWidth - oSrcSizeROI.width. + // nValue The pixel value to be set for border pixels for single channel + // functions. // nppStreamCtx NPP Application Managed Stream Context. nppiCopyConstBorder_32f_C1R_Ctx( static_cast(src.getData()), @@ -67,13 +69,12 @@ void _pad(Tensor & dst, const Tensor & src, static_cast(dst.getData()), dst.getStride(TensorDimension::HEIGHT) * sizeof(float), {static_cast(dst.getWidth()), static_cast(dst.getHeight())}, - topBorderHeight, leftBorderWidth, 0.0, tensor_ops::GetNppStreamContext(stream)); + topBorderHeight, leftBorderWidth, 0.0, cvcore::tensor_ops::GetNppStreamContext(stream)); } -void _threshold(Tensor & dst, const Tensor & src, +void _threshold(Tensor_CHW_C1_F32& dst, const Tensor_CHW_C1_F32& src, float valueLow, float valueHigh, float thresholdValue, - cudaStream_t stream) -{ + cudaStream_t stream) { // Using NPP function: ``nppiThreshold_LTValGTVal_32f_C1R_Ctx'' // Spec ==> NppStatus nppiThreshold_LTValGTVal_32f_C1R_Ctx(const Npp32f * pSrc, // int nSrcStep, @@ -91,8 +92,8 @@ void _threshold(Tensor & dst, const Tensor & src, // nSrcStep Source-Image Stride in bytes. // pDst Destination-Image Pointer. // nDstStep Destination-Image Stride in bytes. - // oSizeROI Size (width, height) of the destination region, i.e. the region that gets - // filled with data from the source image + // oSizeROI Size (width, height) of the destination region, i.e. the region + // that gets filled with data from the source image // nThresholdLT The thresholdLT value. // nValueLT The thresholdLT replacement value. // nThresholdGT The thresholdGT value. @@ -106,11 +107,10 @@ void _threshold(Tensor & dst, const Tensor & src, {static_cast(dst.getWidth()), static_cast(dst.getHeight())}, thresholdValue * (1.0 + std::numeric_limits::epsilon()), valueLow, thresholdValue, valueHigh, - tensor_ops::GetNppStreamContext(stream)); + cvcore::tensor_ops::GetNppStreamContext(stream)); } -void _clear(Tensor & dst, cudaStream_t stream) -{ +void _clear(Tensor_CHW_C1_F32& dst, cudaStream_t stream) { // Using NPP function: ``nppiSet_32f_C1R_Ctx'' // Spec ==> NppStatus nppiSet_32f_C1R_Ctx(const Npp32f nValue // Npp32f * pDst, @@ -122,27 +122,26 @@ void _clear(Tensor & dst, cudaStream_t stream) // nValue The pixel value to be set. // pDst Destination-Image Pointer. // nDstStep Destination-Image Stride in bytes. - // oSizeROI Size (width, height) of the destination region, i.e. the region that gets - // filled with nValue. + // oSizeROI Size (width, height) of the destination region, i.e. the region that + // gets filled with nValue. // nppStreamCtx NPP Application Managed Stream Context. nppiSet_32f_C1R_Ctx( 0.0, static_cast(dst.getData()), dst.getStride(TensorDimension::HEIGHT) * sizeof(float), {static_cast(dst.getWidth()), static_cast(dst.getHeight())}, - tensor_ops::GetNppStreamContext(stream)); + cvcore::tensor_ops::GetNppStreamContext(stream)); } -void _sigmoid(Tensor & dst, Tensor & src, - cudaStream_t stream) -{ - auto context = tensor_ops::GetNppStreamContext(stream); +void _sigmoid(Tensor_CHW_C1_F32& dst, Tensor_CHW_C1_F32& src, + cudaStream_t stream) { + auto context = cvcore::tensor_ops::GetNppStreamContext(stream); // This is a bit of a complex series of operations, here's an exact model // // Explicit: // DST = exp(-1 * ln(1 + exp(-1 * SRC))) - // + // // Properties: // 1) exp(ln(x)) = x // 2) ln(x^y) = y * ln(x) @@ -187,10 +186,12 @@ void _sigmoid(Tensor & dst, Tensor & src, context); } -void _sigmoid(Tensor & dst, Tensor & src) -{ +void _sigmoid(Tensor_CHW_C1_F32& dst, Tensor_CHW_C1_F32& src) { std::transform(src.getData(), src.getData() + src.getDataSize() / sizeof(float), - dst.getData(), [](float value) -> float {return 1.0 / (1.0 + std::exp(-1.0 * value));}); + dst.getData(), [](float value) -> float { + return 1.0 / (1.0 + std::exp(-1.0 * value));}); } -}}} // namespace cvcore::bi3d::detail +} // namespace bi3d +} // namespace isaac +} // namespace nvidia diff --git a/isaac_ros_bi3d/gxf/extensions/bi3d/inference/Bi3D_detail.hpp b/isaac_ros_bi3d/gxf/extensions/bi3d/inference/Bi3D_detail.hpp new file mode 100644 index 0000000..9ab9485 --- /dev/null +++ b/isaac_ros_bi3d/gxf/extensions/bi3d/inference/Bi3D_detail.hpp @@ -0,0 +1,343 @@ +// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES +// Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include + +#include "extensions/tensorops/core/Array.h" +#include "extensions/tensorops/core/BBox.h" +#include "extensions/tensorops/core/ImageUtils.h" +#include "extensions/tensorops/core/NppUtils.h" +#include "extensions/tensorops/core/Tensor.h" +#include "extensions/tensorops/core/Traits.h" + +namespace nvidia { +namespace isaac { +namespace bi3d { + +using TensorLayout = cvcore::tensor_ops::TensorLayout; +using ChannelType = cvcore::tensor_ops::ChannelType; +using ChannelCount = cvcore::tensor_ops::ChannelCount; +using TensorDimension = cvcore::tensor_ops::TensorDimension; +using Tensor_CHW_C1_F32 = cvcore::tensor_ops::Tensor; +using Tensor_HWC_C3_U8 = cvcore::tensor_ops::Tensor; +using Tensor_CHW_CX_F32 = cvcore::tensor_ops::Tensor; +using DisparityLevels = cvcore::tensor_ops::Array; +using BBox = cvcore::tensor_ops::BBox; +using InputImage = Tensor_HWC_C3_U8; +using PreprocessedImage = cvcore::tensor_ops::to_planar_t< + cvcore::tensor_ops::to_f32_t>; +using BatchPreprocessedImage = cvcore::tensor_ops::add_batch_t; +using FeatureResponse = Tensor_CHW_CX_F32; +using BatchFeatureResponse = cvcore::tensor_ops::add_batch_t; +using ConfidenceMap = Tensor_CHW_C1_F32; +using GuidedConfidence = cvcore::tensor_ops::to_cx_t; +using RefinedConfidence = Tensor_CHW_C1_F32; +using DisparityConfidence = cvcore::tensor_ops::to_cx_t; +using QuantizedDisparity = cvcore::tensor_ops::to_u8_t; +using NormalizeFunctor = cvcore::tensor_ops::ImageToNormalizedPlanarTensorOperator< + TensorLayout::HWC, TensorLayout::CHW, ChannelCount::C3, ChannelType::U8>; + +void _pad(Tensor_CHW_C1_F32& dst, const Tensor_CHW_C1_F32& src, + int topBorderHeight, int leftBorderWidth, cudaStream_t stream); + +template +void Pad(cvcore::tensor_ops::Tensor & dst, + cvcore::tensor_ops::Tensor & src, + const cvcore::tensor_ops::Array& topBorderHeights, + const cvcore::tensor_ops::Array& leftBorderWidths, + cudaStream_t stream) { + using TensorType = Tensor_CHW_C1_F32; + + for (std::size_t c = 0; c < dst.getChannelCount(); ++c) { + TensorType _dst{dst.getWidth(), dst.getHeight(), + dst.getData() + c*dst.getStride(TensorDimension::CHANNEL), + dst.isCPU()}; + TensorType _src{src.getWidth(), src.getHeight(), + src.getData() + c*src.getStride(TensorDimension::CHANNEL), + src.isCPU()}; + + _pad(_dst, _src, topBorderHeights[c], leftBorderWidths[c], stream); + } +} + +template +void Pad(cvcore::tensor_ops::Tensor & dst, + cvcore::tensor_ops::Tensor & src, + const cvcore::tensor_ops::Array & topBorderHeights, int leftBorderWidth, + cudaStream_t stream) { + using TensorType = Tensor_CHW_C1_F32; + + for (std::size_t c = 0; c < dst.getChannelCount(); ++c) { + TensorType _dst{dst.getWidth(), dst.getHeight(), + dst.getData() + c*dst.getStride(TensorDimension::CHANNEL), + dst.isCPU()}; + TensorType _src{src.getWidth(), src.getHeight(), + src.getData() + c*src.getStride(TensorDimension::CHANNEL), + src.isCPU()}; + + _pad(_dst, _src, topBorderHeights[c], leftBorderWidth, stream); + } +} + +template +void Pad(cvcore::tensor_ops::Tensor & dst, + cvcore::tensor_ops::Tensor & src, + int topBorderHeight, const cvcore::tensor_ops::Array& leftBorderWidths, + cudaStream_t stream) { + using TensorType = Tensor_CHW_C1_F32; + + for (std::size_t c = 0; c < dst.getChannelCount(); ++c) { + TensorType _dst{dst.getWidth(), dst.getHeight(), + dst.getData() + c*dst.getStride(TensorDimension::CHANNEL), + dst.isCPU()}; + TensorType _src{src.getWidth(), src.getHeight(), + src.getData() + c*src.getStride(TensorDimension::CHANNEL), + src.isCPU()}; + + _pad(_dst, _src, topBorderHeight, leftBorderWidths[c], stream); + } +} + +template +void Pad(cvcore::tensor_ops::Tensor & dst, + cvcore::tensor_ops::Tensor & src, + int topBorderHeight, int leftBorderWidth, cudaStream_t stream) { + using TensorType = Tensor_CHW_C1_F32; + + for (std::size_t c = 0; c < dst.getChannelCount(); ++c) { + TensorType _dst{dst.getWidth(), dst.getHeight(), + dst.getData() + c*dst.getStride(TensorDimension::CHANNEL), + dst.isCPU()}; + TensorType _src{src.getWidth(), src.getHeight(), + src.getData() + c*src.getStride(TensorDimension::CHANNEL), + src.isCPU()}; + + _pad(_dst, _src, topBorderHeight, leftBorderWidth, stream); + } +} + +template +void Pad(cvcore::tensor_ops::Tensor& dst, + cvcore::tensor_ops::Tensor& src, + int topBorderHeight, int leftBorderWidth, cudaStream_t stream) { + using TensorType = Tensor_CHW_CX_F32; + + for (std::size_t n = 0; n < dst.getDepth(); ++n) { + TensorType _dst{dst.getWidth(), dst.getHeight(), dst.getChannelCount(), + dst.getData() + n*dst.getStride(TensorDimension::DEPTH), + dst.isCPU()}; + TensorType _src{src.getWidth(), src.getHeight(), src.getChannelCount(), + src.getData() + n*src.getStride(TensorDimension::DEPTH), + src.isCPU()}; + + Pad(_dst, _src, topBorderHeight, leftBorderWidth, stream); + } +} + +void _threshold(Tensor_CHW_C1_F32 & dst, + const Tensor_CHW_C1_F32 & src, + float valueLow, float valueHigh, float thresholdValue, + cudaStream_t stream); + +template +void Threshold(cvcore::tensor_ops::Tensor & dst, + cvcore::tensor_ops::Tensor & src, + float valueLow, float valueHigh, float thresholdValue, + cudaStream_t stream) { + using TensorType = Tensor_CHW_C1_F32; + + TensorType _dst{dst.getWidth(), dst.getHeight() * dst.getChannelCount(), + dst.getData(), dst.isCPU()}; + TensorType _src{src.getWidth(), src.getHeight() * src.getChannelCount(), + src.getData(), src.isCPU()}; + + _threshold(_dst, _src, valueLow, valueHigh, thresholdValue, stream); +} + +template +void Threshold(cvcore::tensor_ops::Tensor & dst, + cvcore::tensor_ops::Tensor & src, + float valueLow, float valueHigh, float thresholdValue, + cudaStream_t stream) { + using TensorType = Tensor_CHW_CX_F32; + + TensorType _dst{dst.getWidth(), dst.getHeight(), + dst.getChannelCount() * dst.getDepth(), + dst.getData(), dst.isCPU()}; + TensorType _src{src.getWidth(), src.getHeight(), + src.getChannelCount() * src.getDepth(), + src.getData(), src.isCPU()}; + + Threshold(_dst, _src, valueLow, valueHigh, thresholdValue, stream); +} + +template +void CropAndResize(cvcore::tensor_ops::Tensor & dst, + cvcore::tensor_ops::Tensor & src, + const BBox & dstROI, const BBox & srcROI, + cvcore::tensor_ops::InterpolationType type, cudaStream_t stream) { + using TensorType = Tensor_CHW_C1_F32; + + for (std::size_t c = 0; c < dst.getChannelCount(); ++c) { + TensorType _dst{dst.getWidth(), dst.getHeight(), + dst.getData() + c*dst.getStride(TensorDimension::CHANNEL), + dst.isCPU()}; + TensorType _src{src.getWidth(), src.getHeight(), + src.getData() + c*src.getStride(TensorDimension::CHANNEL), + src.isCPU()}; + + cvcore::tensor_ops::CropAndResize(_dst, _src, dstROI, srcROI, type, stream); + } +} + +template +void CropAndResize(cvcore::tensor_ops::Tensor & dst, + cvcore::tensor_ops::Tensor & src, + const BBox & dstROI, const BBox & srcROI, + cvcore::tensor_ops::InterpolationType type, cudaStream_t stream) { + using TensorType = Tensor_CHW_CX_F32; + + for (std::size_t n = 0; n < dst.getDepth(); ++n) { + TensorType _dst{dst.getWidth(), dst.getHeight(), dst.getChannelCount(), + dst.getData() + n*dst.getStride(TensorDimension::DEPTH), + dst.isCPU()}; + TensorType _src{src.getWidth(), src.getHeight(), src.getChannelCount(), + src.getData() + n*src.getStride(TensorDimension::DEPTH), + src.isCPU()}; + + CropAndResize(_dst, _src, dstROI, srcROI, type, stream); + } +} + +template +void Resize(cvcore::tensor_ops::Tensor & dst, + cvcore::tensor_ops::Tensor & src, + cvcore::tensor_ops::InterpolationType type, cudaStream_t stream) { + using TensorType = Tensor_CHW_C1_F32; + + for (std::size_t c = 0; c < dst.getChannelCount(); ++c) { + TensorType _dst{dst.getWidth(), dst.getHeight(), + dst.getData() + c*dst.getStride(TensorDimension::CHANNEL), + dst.isCPU()}; + TensorType _src{src.getWidth(), src.getHeight(), + src.getData() + c*src.getStride(TensorDimension::CHANNEL), + src.isCPU()}; + + Resize(_dst, _src, false, type, stream); + } +} + +template +void Resize(cvcore::tensor_ops::Tensor & dst, + cvcore::tensor_ops::Tensor & src, + cvcore::tensor_ops::InterpolationType type, cudaStream_t stream) { + using TensorType = Tensor_CHW_CX_F32; + + for (std::size_t n = 0; n < dst.getDepth(); ++n) { + TensorType _dst{dst.getWidth(), dst.getHeight(), dst.getChannelCount(), + dst.getData() + n*dst.getStride(TensorDimension::DEPTH), + dst.isCPU()}; + TensorType _src{src.getWidth(), src.getHeight(), src.getChannelCount(), + src.getData() + n*src.getStride(TensorDimension::DEPTH), + src.isCPU()}; + + Resize(_dst, _src, type, stream); + } +} + +void _clear(Tensor_CHW_C1_F32& dst, cudaStream_t stream); + +template +void Clear(cvcore::tensor_ops::Tensor & dst, + cudaStream_t stream) { + using TensorType = Tensor_CHW_C1_F32; + + for (std::size_t c = 0; c < dst.getChannelCount(); ++c) { + TensorType _dst{dst.getWidth(), dst.getHeight(), + dst.getData() + c*dst.getStride(TensorDimension::CHANNEL), + dst.isCPU()}; + + _clear(_dst, stream); + } +} + +template +void ConvertBitDepth(cvcore::tensor_ops::Tensor & dst, + cvcore::tensor_ops::Tensor & src, + float scale, cudaStream_t stream) { + using TensorSrcType = Tensor_CHW_C1_F32; + using TensorDstType = cvcore::tensor_ops::to_u8_t; + + for (std::size_t c = 0; c < dst.getChannelCount(); ++c) { + TensorDstType _dst{dst.getWidth(), dst.getHeight(), + dst.getData() + c*dst.getStride(TensorDimension::CHANNEL), + dst.isCPU()}; + TensorSrcType _src{src.getWidth(), src.getHeight(), + src.getData() + c*src.getStride(TensorDimension::CHANNEL), + src.isCPU()}; + + ConvertBitDepth(_dst, _src, scale, stream); + } +} + +void _sigmoid(Tensor_CHW_C1_F32& dst, + Tensor_CHW_C1_F32& src, cudaStream_t stream); + +void _sigmoid(Tensor_CHW_C1_F32& dst, Tensor_CHW_C1_F32& src); + +template +void Sigmoid(cvcore::tensor_ops::Tensor & dst, + cvcore::tensor_ops::Tensor & src, + cudaStream_t stream) { + using TensorType = Tensor_CHW_C1_F32; + + for (std::size_t c = 0; c < dst.getChannelCount(); ++c) { + TensorType _dst{dst.getWidth(), dst.getHeight(), + dst.getData() + c*dst.getStride(TensorDimension::CHANNEL), + dst.isCPU()}; + TensorType _src{src.getWidth(), src.getHeight(), + src.getData() + c*src.getStride(TensorDimension::CHANNEL), + src.isCPU()}; + + _sigmoid(_dst, _src, stream); + } +} + +template +void Sigmoid(cvcore::tensor_ops::Tensor& dst, + cvcore::tensor_ops::Tensor& src) { + + for (std::size_t c = 0; c < dst.getChannelCount(); ++c) { + Tensor_CHW_C1_F32 _dst{dst.getWidth(), dst.getHeight(), + dst.getData() + c*dst.getStride(TensorDimension::CHANNEL), + dst.isCPU()}; + Tensor_CHW_C1_F32 _src{src.getWidth(), src.getHeight(), + src.getData() + c*src.getStride(TensorDimension::CHANNEL), + src.isCPU()}; + + _sigmoid(_dst, _src); + } +} + +} // namespace bi3d +} // namespace isaac +} // namespace nvidia diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/core/Model.h b/isaac_ros_bi3d/gxf/extensions/bi3d/inference/Model.h similarity index 79% rename from isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/core/Model.h rename to isaac_ros_bi3d/gxf/extensions/bi3d/inference/Model.h index 4a14945..ac17511 100644 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/core/Model.h +++ b/isaac_ros_bi3d/gxf/extensions/bi3d/inference/Model.h @@ -15,36 +15,36 @@ // // SPDX-License-Identifier: Apache-2.0 -#ifndef CVCORE_MODEL_H -#define CVCORE_MODEL_H +#pragma once +#include #include -#include "Image.h" +#include "extensions/tensorops/core/Image.h" -namespace cvcore { +namespace nvidia { +namespace isaac { +namespace bi3d { /** * Struct to describe input type required by the model */ -struct ModelInputParams -{ +struct ModelInputParams { size_t maxBatchSize; /**< maxbatchSize supported by network*/ size_t inputLayerWidth; /**< Input layer width */ size_t inputLayerHeight; /**< Input layer Height */ - ImageType modelInputType; /**< Input Layout type */ + cvcore::tensor_ops::ImageType modelInputType; /**< Input Layout type */ }; /** * Struct to describe the model */ -struct ModelInferenceParams -{ +struct ModelInferenceParams { std::string engineFilePath; /**< Engine file path. */ std::vector inputLayers; /**< names of input layers. */ std::vector outputLayers; /**< names of output layers. */ }; -} // namespace cvcore - -#endif // CVCORE_MODEL_H +} // namespace bi3d +} // namespace isaac +} // namespace nvidia diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/inferencer/Errors.cpp b/isaac_ros_bi3d/gxf/gems/dnn_inferencer/inferencer/Errors.cpp similarity index 86% rename from isaac_ros_bi3d/gxf/bi3d/cvcore/src/inferencer/Errors.cpp rename to isaac_ros_bi3d/gxf/gems/dnn_inferencer/inferencer/Errors.cpp index f6135a9..2fe26d6 100644 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/inferencer/Errors.cpp +++ b/isaac_ros_bi3d/gxf/gems/dnn_inferencer/inferencer/Errors.cpp @@ -15,34 +15,31 @@ // // SPDX-License-Identifier: Apache-2.0 -#include "cv/inferencer/Errors.h" +#include +#include "gems/dnn_inferencer/inferencer/Errors.h" #ifndef __cpp_lib_to_underlying namespace std { template -constexpr underlying_type_t to_underlying(Enum e) noexcept -{ +constexpr underlying_type_t to_underlying(Enum e) noexcept { return static_cast>(e); } }; // namespace std -#endif // __cpp_lib_to_underlying +#endif // __cpp_lib_to_underlying -namespace cvcore { namespace inferencer { +namespace cvcore { +namespace inferencer { namespace detail { -struct InferencerErrorCategory : std::error_category -{ - virtual const char *name() const noexcept override final - { +struct InferencerErrorCategory : std::error_category { + const char* name() const noexcept final { return "cvcore-inferencer-error"; } - virtual std::string message(int value) const override final - { + std::string message(int value) const final { std::string result; - switch (value) - { + switch (value) { case std::to_underlying(InferencerErrorCode::SUCCESS): result = "(SUCCESS) No errors detected"; break; @@ -53,10 +50,12 @@ struct InferencerErrorCategory : std::error_category result = "(INVALID_OPERATION) Invalid operation performed"; break; case std::to_underlying(InferencerErrorCode::TRITON_SERVER_NOT_READY): - result = "(TRITON_SERVER_NOT_READY) Triton server is not live or the serverUrl is incorrect"; + result = "(TRITON_SERVER_NOT_READY) Triton server is not live or the serverUrl" + "is incorrect"; break; case std::to_underlying(InferencerErrorCode::TRITON_CUDA_SHARED_MEMORY_ERROR): - result = "(TRITON_CUDA_SHARED_MEMORY_ERROR) Unable to map/unmap cuda shared memory for triton server"; + result = "(TRITON_CUDA_SHARED_MEMORY_ERROR) Unable to map/unmap cuda shared memory" + "for triton server"; break; case std::to_underlying(InferencerErrorCode::TRITON_INFERENCE_ERROR): result = "(TRITON_INFERENCE_ERROR) Error during inference using triton API"; @@ -77,12 +76,11 @@ struct InferencerErrorCategory : std::error_category return result; } - virtual std::error_condition default_error_condition(int code) const noexcept override final - { + std::error_condition default_error_condition(int code) const noexcept final { std::error_condition result; + using ErrorCode = cvcore::tensor_ops::ErrorCode; - switch (code) - { + switch (code) { case std::to_underlying(InferencerErrorCode::SUCCESS): result = ErrorCode::SUCCESS; break; @@ -118,12 +116,13 @@ struct InferencerErrorCategory : std::error_category return result; } }; -} // namespace detail +} // namespace detail const detail::InferencerErrorCategory errorCategory{}; -std::error_code make_error_code(InferencerErrorCode ec) noexcept -{ +std::error_code make_error_code(InferencerErrorCode ec) noexcept { return {std::to_underlying(ec), errorCategory}; } -}} // namespace cvcore::inferencer + +} // namespace inferencer +} // namespace cvcore diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/inferencer/Errors.h b/isaac_ros_bi3d/gxf/gems/dnn_inferencer/inferencer/Errors.h similarity index 77% rename from isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/inferencer/Errors.h rename to isaac_ros_bi3d/gxf/gems/dnn_inferencer/inferencer/Errors.h index c569f10..39da91f 100644 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/inferencer/Errors.h +++ b/isaac_ros_bi3d/gxf/gems/dnn_inferencer/inferencer/Errors.h @@ -15,18 +15,17 @@ // // SPDX-License-Identifier: Apache-2.0 -#ifndef CVCORE_ERRORS_H -#define CVCORE_ERRORS_H +#pragma once -#include "cv/core/CVError.h" +#include "extensions/tensorops/core/CVError.h" -namespace cvcore { namespace inferencer { +namespace cvcore { +namespace inferencer { /* * Enum class describing the inference error codes. */ -enum class InferencerErrorCode : std::int32_t -{ +enum class InferencerErrorCode : std::int32_t { SUCCESS = 0, INVALID_ARGUMENT, INVALID_OPERATION, @@ -36,23 +35,24 @@ enum class InferencerErrorCode : std::int32_t TRITON_INFERENCE_ERROR, TRITON_REGISTER_LAYER_ERROR, TENSORRT_INFERENCE_ERROR, + TENSORRT_ENGINE_ERROR }; -}} // namespace cvcore::inferencer +} // namespace inferencer +} // namespace cvcore namespace std { template<> -struct is_error_code_enum : true_type -{ +struct is_error_code_enum : true_type { }; -} // namespace std +} // namespace std -namespace cvcore { namespace inferencer { +namespace cvcore { +namespace inferencer { std::error_code make_error_code(InferencerErrorCode) noexcept; -}} // namespace cvcore::inferencer - -#endif // CVCORE_ERRORS_H +} // namespace inferencer +} // namespace cvcore diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/inferencer/IInferenceBackend.h b/isaac_ros_bi3d/gxf/gems/dnn_inferencer/inferencer/IInferenceBackend.h similarity index 71% rename from isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/inferencer/IInferenceBackend.h rename to isaac_ros_bi3d/gxf/gems/dnn_inferencer/inferencer/IInferenceBackend.h index 7213a06..9212e5e 100644 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/inferencer/IInferenceBackend.h +++ b/isaac_ros_bi3d/gxf/gems/dnn_inferencer/inferencer/IInferenceBackend.h @@ -15,8 +15,7 @@ // // SPDX-License-Identifier: Apache-2.0 -#ifndef CVCORE_IINFERENCEBACKEND_H -#define CVCORE_IINFERENCEBACKEND_H +#pragma once #include #include @@ -26,38 +25,37 @@ #include #include -#include "cv/core/Tensor.h" +#include +#include "extensions/tensorops/core/Tensor.h" -namespace cvcore { namespace inferencer { +namespace cvcore { +namespace inferencer { /** * Struct type to describe input and output layers. */ -struct LayerInfo -{ +struct LayerInfo { size_t index; /**< Block Index of layer */ std::string name; /**< Name of layer */ std::vector shape; /**< Shape of layer */ - cvcore::ChannelType dataType; /**< Datatype of layer */ - cvcore::TensorLayout layout; /**< Tensor layour of layer */ + cvcore::tensor_ops::ChannelType dataType; /**< Datatype of layer */ + cvcore::tensor_ops::TensorLayout layout; /**< Tensor layour of layer */ size_t layerSize; }; /** * Enum class to describe the backend protocol for triton */ -enum class BackendProtocol -{ - GRPC, /**< GRPC protocol */ - HTTP /**< HTTP Protocol */ +enum class BackendProtocol { + GRPC, + HTTP }; /** * Struct type to describe the input for triton inference. */ -struct TritonRemoteInferenceParams -{ - std::string serverUrl; /**< Server url created by running the triton server for a give model path */ +struct TritonRemoteInferenceParams { + std::string serverUrl; /**< Server url created by running the triton server */ bool verbose; /**< Verbose log from backend */ BackendProtocol protocolType; /**< Backend protocol type */ std::string modelName; /**< Model name as per model respoitory */ @@ -67,11 +65,10 @@ struct TritonRemoteInferenceParams /** * Struct type to describe the model metadata parsed by the inference backend. */ -struct ModelMetaData -{ +struct ModelMetaData { std::string modelName; /**< Model name */ std::string modelVersion; /**< Model version */ - std::unordered_map inputLayers; /**< Map of input layer information indexed by layer name */ + std::unordered_map inputLayers; /**< Map of input layer */ std::unordered_map outputLayers; /**< Map of output layer information indexed by layer name */ size_t maxBatchSize; /**< Maximum batch size */ @@ -80,8 +77,7 @@ struct ModelMetaData /** * Enum type for TensorRT inference type */ -enum class TRTInferenceType -{ +enum class TRTInferenceType { TRT_ENGINE, /**< Inference using TRT engine file */ TRT_ENGINE_IN_MEMORY /**< Inference using TRT Cuda Engine */ }; @@ -89,26 +85,37 @@ enum class TRTInferenceType /** * TRT Logger */ -class TRTLogger : public nvinfer1::ILogger -{ -public: - void log(Severity severity, const char *msg) noexcept - { - if ((severity == Severity::kERROR) || (severity == Severity::kWARNING)) - { +class TRTLogger : public nvinfer1::ILogger { + public: + void log(Severity severity, const char* msg) noexcept { + if ((severity == Severity::kERROR) || (severity == Severity::kWARNING)) { std::cerr << msg << std::endl; } } }; +/** + * Enum class to describe the model build flags + */ +enum OnnxModelBuildFlag { + NONE = 0, + kINT8 = 1, + kFP16 = 2, + kGPU_FALLBACK = 4, +}; + /** * Struct type to describe the input for triton inference. */ -struct TensorRTInferenceParams -{ +struct TensorRTInferenceParams { TRTInferenceType inferType; /**< TensorRT inference type */ - nvinfer1::ICudaEngine *engine; /**< Cuda engine file for TRT_ENGINE_IN_MEMORY type. Nullptr if TRT_ENGINE is used */ - std::string engineFilePath; /**< Engine file path for TRT_ENGINE type. Set to null otherwise */ + nvinfer1::ICudaEngine* engine; /**< Cuda engine file for TRT_ENGINE_IN_MEMORY type. */ + // Nullptr if TRT_ENGINE is used + std::string onnxFilePath; /**< ONNX model file path. */ + std::string engineFilePath; /**< Engine file path for TRT_ENGINE type. */ + bool force_engine_update; + int32_t buildFlags; + int64_t max_workspace_size; std::size_t maxBatchSize; /**< Max Batch size */ std::vector inputLayerNames; /** Input layer names */ std::vector outputLayerNames; /** Output layer names */ @@ -118,9 +125,8 @@ struct TensorRTInferenceParams /** * Interface for TritonRemote , Triton C and Native TensorRT inference backend. */ -class IInferenceBackendClient -{ -public: +class IInferenceBackendClient { + public: virtual ~IInferenceBackendClient() noexcept = default; /** @@ -129,7 +135,8 @@ class IInferenceBackendClient * @param inputLayerName Input Layer name * @return error code */ - virtual std::error_code setInput(const cvcore::TensorBase &trtInputBuffer, std::string inputLayerName) = 0; + virtual std::error_code setInput(const cvcore::tensor_ops::TensorBase& trtInputBuffer, + std::string inputLayerName) = 0; /** * Function to set output layer data @@ -137,7 +144,8 @@ class IInferenceBackendClient * @param outputLayerName Output Layer name * @return error code */ - virtual std::error_code setOutput(cvcore::TensorBase &trtOutputBuffer, std::string outputLayerName) = 0; + virtual std::error_code setOutput(cvcore::tensor_ops::TensorBase& trtOutputBuffer, + std::string outputLayerName) = 0; /** * Returns the model metadata parsed by the inference backend. @@ -147,7 +155,7 @@ class IInferenceBackendClient /** * Inference based on input and output set. enqueueV2 for TensorRT and inferSync for Triton. - * @param Batch size of input for inference. Default set to 1. Must be <= Max Batch Size used for buffers. + * @param Batch size of input for inference. Default set to 1. Must be <= Max Batch Size . * @return error code */ virtual std::error_code infer(size_t batchSize = 1) = 0; @@ -157,7 +165,7 @@ class IInferenceBackendClient * @param cudaStream_t cuda input stream * @return error code */ - virtual std::error_code setCudaStream(cudaStream_t) = 0; // Only in TRT + virtual std::error_code setCudaStream(cudaStream_t) = 0; /** * Unregisters the tensor mapped from the inference backend @@ -172,14 +180,15 @@ class IInferenceBackendClient */ virtual std::error_code unregister() = 0; -protected: + protected: IInferenceBackendClient() = default; - IInferenceBackendClient(const IInferenceBackendClient &) = default; - IInferenceBackendClient &operator=(const IInferenceBackendClient &) = default; + IInferenceBackendClient(const IInferenceBackendClient&) = default; + IInferenceBackendClient& operator=(const IInferenceBackendClient&) = default; IInferenceBackendClient(IInferenceBackendClient &&) noexcept = default; - IInferenceBackendClient &operator=(IInferenceBackendClient &&) noexcept = default; + IInferenceBackendClient& operator=(IInferenceBackendClient &&) noexcept = default; }; using InferenceBackendClient = IInferenceBackendClient *; -}} // namespace cvcore::inferencer -#endif + +} // namespace inferencer +} // namespace cvcore diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/inferencer/Inferencer.cpp b/isaac_ros_bi3d/gxf/gems/dnn_inferencer/inferencer/Inferencer.cpp similarity index 69% rename from isaac_ros_bi3d/gxf/bi3d/cvcore/src/inferencer/Inferencer.cpp rename to isaac_ros_bi3d/gxf/gems/dnn_inferencer/inferencer/Inferencer.cpp index 2ce6a2d..ba4a590 100644 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/inferencer/Inferencer.cpp +++ b/isaac_ros_bi3d/gxf/gems/dnn_inferencer/inferencer/Inferencer.cpp @@ -15,16 +15,22 @@ // // SPDX-License-Identifier: Apache-2.0 -#include "cv/inferencer/Inferencer.h" +#include "gems/dnn_inferencer/inferencer/Inferencer.h" #include +#include +#include +#include -#include "cv/inferencer/Errors.h" -#include "cv/inferencer/IInferenceBackend.h" -#include "tensorrt/TensorRTInferencer.h" -#include "triton/TritonGrpcInferencer.h" +#include "Errors.h" +#include "IInferenceBackend.h" +#include "TensorRTInferencer.h" +#include "TritonGrpcInferencer.h" + +namespace cvcore { +namespace inferencer { +using ErrorCode = cvcore::tensor_ops::ErrorCode; -namespace cvcore { namespace inferencer { std::mutex InferenceBackendFactory::inferenceMutex; #ifdef ENABLE_TRITON @@ -32,52 +38,42 @@ std::unordered_map> InferenceBackendFactory::tritonRemoteMap; std::error_code InferenceBackendFactory::CreateTritonRemoteInferenceBackendClient( - InferenceBackendClient &client, const TritonRemoteInferenceParams ¶ms) -{ + InferenceBackendClient& client, const TritonRemoteInferenceParams& params) { std::lock_guard instanceLock(inferenceMutex); - if (params.protocolType == BackendProtocol::HTTP) - { + if (params.protocolType == BackendProtocol::HTTP) { return ErrorCode::NOT_IMPLEMENTED; } std::error_code result = ErrorCode::SUCCESS; std::string hashString = params.serverUrl + params.modelName + params.modelVersion; - try - { - if (tritonRemoteMap.find(hashString) != tritonRemoteMap.end()) - { + try { + if (tritonRemoteMap.find(hashString) != tritonRemoteMap.end()) { client = tritonRemoteMap[hashString].second; tritonRemoteMap[hashString].first++; - } - else - { + } else { tritonRemoteMap[hashString] = - std::make_pair(1, new TritonGrpcInferencer(params)); + std::pair(1, + new TritonGrpcInferencer(params)); } } - catch (std::error_code &e) - { + catch (std::error_code &e) { result = e; } - catch (...) - { + catch (...) { result = ErrorCode::INVALID_ARGUMENT; } client = tritonRemoteMap[hashString].second; return result; } -std::error_code InferenceBackendFactory::DestroyTritonRemoteInferenceBackendClient(InferenceBackendClient &client) -{ +std::error_code InferenceBackendFactory::DestroyTritonRemoteInferenceBackendClient( + InferenceBackendClient& client) { std::lock_guard instanceLock(inferenceMutex); - for (auto &it : tritonRemoteMap) - { - if (it.second.second == client) - { + for (auto &it : tritonRemoteMap) { + if (it.second.second == client) { it.second.first--; - if (it.second.first == 0) - { + if (it.second.first == 0) { tritonRemoteMap.erase(it.first); client->unregister(); delete client; @@ -91,33 +87,28 @@ std::error_code InferenceBackendFactory::DestroyTritonRemoteInferenceBackendClie } #endif -std::error_code InferenceBackendFactory::CreateTensorRTInferenceBackendClient(InferenceBackendClient &client, - const TensorRTInferenceParams ¶ms) -{ +std::error_code InferenceBackendFactory::CreateTensorRTInferenceBackendClient( + InferenceBackendClient& client, const TensorRTInferenceParams& params) { std::lock_guard instanceLock(inferenceMutex); std::error_code result = ErrorCode::SUCCESS; - try - { + try { client = new TensorRTInferencer(params); } - catch (std::error_code &e) - { + catch (std::error_code &e) { result = e; } - catch (...) - { + catch (...) { result = ErrorCode::INVALID_ARGUMENT; } return result; } -std::error_code InferenceBackendFactory::DestroyTensorRTInferenceBackendClient(InferenceBackendClient &client) -{ +std::error_code InferenceBackendFactory::DestroyTensorRTInferenceBackendClient( + InferenceBackendClient& client) { std::lock_guard instanceLock(inferenceMutex); - if (client != nullptr) - { + if (client != nullptr) { client->unregister(); delete client; client = nullptr; @@ -126,5 +117,5 @@ std::error_code InferenceBackendFactory::DestroyTensorRTInferenceBackendClient(I return ErrorCode::SUCCESS; } - -}} // namespace cvcore::inferencer +} // namespace inferencer +} // namespace cvcore diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/inferencer/Inferencer.h b/isaac_ros_bi3d/gxf/gems/dnn_inferencer/inferencer/Inferencer.h similarity index 69% rename from isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/inferencer/Inferencer.h rename to isaac_ros_bi3d/gxf/gems/dnn_inferencer/inferencer/Inferencer.h index ad61131..d07de15 100644 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/include/cv/inferencer/Inferencer.h +++ b/isaac_ros_bi3d/gxf/gems/dnn_inferencer/inferencer/Inferencer.h @@ -15,65 +15,72 @@ // // SPDX-License-Identifier: Apache-2.0 -#ifndef CVCORE_INFERENCER_H -#define CVCORE_INFERENCER_H +#pragma once #include #include +#include #include +#include -#include "cv/core/Tensor.h" -#include "cv/inferencer/IInferenceBackend.h" +#include "extensions/tensorops/core/Tensor.h" +#include "IInferenceBackend.h" -namespace cvcore { namespace inferencer { +namespace cvcore { +namespace inferencer { /** * A class to create and destroy a client for a given inference backend type */ -class InferenceBackendFactory -{ -public: +class InferenceBackendFactory { + public: #ifdef ENABLE_TRITON /** - * Function to create client for Triton remote inference backend based on the Triton remote inference paramaters. + * Function to create client for Triton remote inference backend based on the Triton remote + * inference paramaters. * @param client client object created * @param Triton remote inference config parameters. * @return error code */ - static std::error_code CreateTritonRemoteInferenceBackendClient(InferenceBackendClient &client, - const TritonRemoteInferenceParams &); + static std::error_code CreateTritonRemoteInferenceBackendClient(InferenceBackendClient& client, + const TritonRemoteInferenceParams&); /** * Function to Destroy the triton grpc client * @param client client object created * @return error code */ - static std::error_code DestroyTritonRemoteInferenceBackendClient(InferenceBackendClient &client); + static std::error_code DestroyTritonRemoteInferenceBackendClient( + InferenceBackendClient& client); #endif /** - * Function to create client for TensorRT inference backend based on the TensorRT inference paramaters. + * Function to create client for TensorRT inference backend based on the TensorRT + * inference paramaters. * @param client client object created * @param TensorRT inference config parameters. * @return error code */ - static std::error_code CreateTensorRTInferenceBackendClient(InferenceBackendClient &client, - const TensorRTInferenceParams &); + static std::error_code CreateTensorRTInferenceBackendClient(InferenceBackendClient& client, + const TensorRTInferenceParams&); /** * Function to Destroy the tensorrt client * @param client client object created * @return error code */ - static std::error_code DestroyTensorRTInferenceBackendClient(InferenceBackendClient &client); + static std::error_code DestroyTensorRTInferenceBackendClient(InferenceBackendClient& client); -private: + private: #ifdef ENABLE_TRITON - // Keeps track of any changes in url/model repo path and returns an existing / new client instance. - static std::unordered_map> tritonRemoteMap; + // Keeps track of any changes in url/model repo path and returns an existing / new client + // instance. + static std::unordered_map> tritonRemoteMap; #endif static std::mutex inferenceMutex; }; -}} // namespace cvcore::inferencer -#endif + +} // namespace inferencer +} // namespace cvcore diff --git a/isaac_ros_bi3d/gxf/gems/dnn_inferencer/inferencer/TensorRTInferencer.cpp b/isaac_ros_bi3d/gxf/gems/dnn_inferencer/inferencer/TensorRTInferencer.cpp new file mode 100644 index 0000000..62f9c05 --- /dev/null +++ b/isaac_ros_bi3d/gxf/gems/dnn_inferencer/inferencer/TensorRTInferencer.cpp @@ -0,0 +1,387 @@ +// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES +// Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 +#include "gems/dnn_inferencer/inferencer/TensorRTInferencer.h" +#include +#include +#include +#include +#include +#include + +#include +#include "gems/dnn_inferencer/inferencer/Errors.h" +#include "gems/dnn_inferencer/inferencer/IInferenceBackend.h" +#include "gems/dnn_inferencer/inferencer/Inferencer.h" +#include "gems/dnn_inferencer/inferencer/TensorRTUtils.h" +#include "gxf/core/expected.hpp" + +namespace cvcore { +namespace inferencer { + +namespace { +// using namespace cvcore::tensor_ops; +using TensorBase = cvcore::tensor_ops::TensorBase; +using ChannelType = cvcore::tensor_ops::ChannelType; +using ChannelCount = cvcore::tensor_ops::ChannelCount; +using ErrorCode = cvcore::tensor_ops::ErrorCode; +size_t getDataSize(const std::vector& shape, ChannelType dataType) { + size_t layerShape = 1; + for (size_t k = 0; k < shape.size(); k++) + layerShape *= shape[k] <= 0 ? 1 : shape[k]; + + return layerShape * GetChannelSize(dataType); +} +} // namespace + +std::error_code TensorRTInferencer::getLayerInfo(LayerInfo& layer, std::string layerName) { + layer.name = layerName; + layer.index = m_inferenceEngine->getBindingIndex(layerName.c_str()); + auto dim = m_inferenceEngine->getBindingDimensions(layer.index); + nvinfer1::TensorFormat tensorFormat = m_inferenceEngine->getBindingFormat(layer.index); + + std::error_code err; + err = getCVCoreChannelLayoutFromTensorRT(layer.layout, tensorFormat); + if (err != make_error_code(ErrorCode::SUCCESS)) { + return ErrorCode::INVALID_ARGUMENT; + } + + for (int32_t cnt = 0; cnt < dim.nbDims; cnt++) { + layer.shape.push_back(dim.d[cnt]); + } + + err = getCVCoreChannelTypeFromTensorRT(layer.dataType, + m_inferenceEngine->getBindingDataType(layer.index)); + layer.layerSize = getDataSize(layer.shape, layer.dataType); + if (err != make_error_code(ErrorCode::SUCCESS)) { + return ErrorCode::INVALID_ARGUMENT; + } + + return ErrorCode::SUCCESS; +} + +std::error_code TensorRTInferencer::ParseTRTModel() { + m_modelInfo.modelName = m_inferenceEngine->getName(); + m_modelInfo.modelVersion = ""; + m_modelInfo.maxBatchSize = m_maxBatchSize; + std::error_code err; + for (size_t i = 0; i < m_inputLayers.size(); i++) { + LayerInfo layer; + err = getLayerInfo(layer, m_inputLayers[i]); + if (err != make_error_code(ErrorCode::SUCCESS)) { + return err; + } + m_modelInfo.inputLayers[layer.name] = layer; + } + for (size_t i = 0; i < m_outputLayers.size(); i++) { + LayerInfo layer; + err = getLayerInfo(layer, m_outputLayers[i]); + if (err != make_error_code(ErrorCode::SUCCESS)) { + return err; + } + m_modelInfo.outputLayers[layer.name] = layer; + } + + return ErrorCode::SUCCESS; +} + +std::error_code TensorRTInferencer::convertModelToEngine(int32_t dla_core, + const char* model_file, int64_t max_workspace_size, int32_t buildFlags, + std::size_t max_batch_size) { + GXF_LOG_INFO("Convert to engine from onnx file: %s", model_file); + // Creates the engine Builder + std::unique_ptr builder(nvinfer1::createInferBuilder(*(m_logger.get()))); + + // Builder Config provides options to the Builder + std::unique_ptr builderConfig(builder->createBuilderConfig()); + builderConfig->setMemoryPoolLimit(nvinfer1::MemoryPoolType::kWORKSPACE, max_workspace_size); + + if (dla_core >= 0) { + builderConfig->setDefaultDeviceType(nvinfer1::DeviceType::kDLA); + builderConfig->setDLACore(dla_core); + } + if (buildFlags & kINT8) { + builderConfig->setFlag(nvinfer1::BuilderFlag::kINT8); + builderConfig->setInt8Calibrator(nullptr); + } + if (buildFlags & OnnxModelBuildFlag::kFP16) { + builderConfig->setFlag(nvinfer1::BuilderFlag::kFP16); + } + + if (buildFlags & kGPU_FALLBACK) { + builderConfig->setFlag(nvinfer1::BuilderFlag::kGPU_FALLBACK); + } + + // Parses ONNX with explicit batch size for support of dynamic shapes/batch + std::unique_ptr network(builder->createNetworkV2( + 1U << static_cast(nvinfer1::NetworkDefinitionCreationFlag::kEXPLICIT_BATCH))); + + std::unique_ptr onnx_parser( + nvonnxparser::createParser(*network, *(m_logger.get()))); + if (!onnx_parser->parseFromFile(model_file, + static_cast(nvinfer1::ILogger::Severity::kWARNING))) { + GXF_LOG_ERROR("Failed to parse ONNX file %s", model_file); + return ErrorCode::INVALID_ARGUMENT; + } + + // Provides optimization profile for dynamic size input bindings + nvinfer1::IOptimizationProfile* optimization_profile = builder->createOptimizationProfile(); + // Checks input dimensions and adds to optimization profile if needed + const int number_inputs = network->getNbInputs(); + for (int i = 0; i < number_inputs; ++i) { + auto* bind_tensor = network->getInput(i); + const char* bind_name = bind_tensor->getName(); + nvinfer1::Dims dims = bind_tensor->getDimensions(); + + // Validates binding info + if (dims.nbDims <= 0) { + GXF_LOG_ERROR("Invalid input tensor dimensions for binding %s", bind_name); + return ErrorCode::INVALID_ARGUMENT; + } + for (int j = 1; j < dims.nbDims; ++j) { + if (dims.d[j] <= 0) { + GXF_LOG_ERROR( + "Input binding %s requires dynamic size on dimension No.%d which is not supported", + bind_tensor->getName(), j); + return ErrorCode::INVALID_ARGUMENT; + } + } + if (dims.d[0] == -1) { + // Only case with first dynamic dimension is supported and assumed to be batch size. + // Always optimizes for 1-batch. + dims.d[0] = 1; + optimization_profile->setDimensions(bind_name, nvinfer1::OptProfileSelector::kMIN, dims); + optimization_profile->setDimensions(bind_name, nvinfer1::OptProfileSelector::kOPT, dims); + dims.d[0] = max_batch_size; + if (max_batch_size <= 0) { + dims.d[0] = 1; + } + optimization_profile->setDimensions(bind_name, nvinfer1::OptProfileSelector::kMAX, dims); + } + } + builderConfig->addOptimizationProfile(optimization_profile); + + // Creates TensorRT Engine Plan + std::unique_ptr engine( + builder->buildEngineWithConfig(*network, *builderConfig)); + if (!engine) { + GXF_LOG_ERROR("Failed to build TensorRT engine from model %s.", model_file); + return InferencerErrorCode::INVALID_ARGUMENT; + } + + std::unique_ptr model_stream(engine->serialize()); + if (!model_stream || model_stream->size() == 0 || model_stream->data() == nullptr) { + GXF_LOG_ERROR("Fail to serialize TensorRT Engine."); + return InferencerErrorCode::INVALID_ARGUMENT; + } + + // Prepares return value + const char* data = static_cast(model_stream->data()); + m_modelEngineStream.resize(model_stream->size()); + std::copy(data, data + model_stream->size(), m_modelEngineStream.data()); + m_modelEngineStreamSize = model_stream->size(); + return InferencerErrorCode::SUCCESS; +} + +// Writes engine plan to specified file path +std::error_code SerializeEnginePlan(const std::vector& plan, const std::string path) { + // Write Plan To Disk + std::ofstream out_stream(path.c_str(), std::ofstream::binary); + if (!out_stream.is_open()) { + GXF_LOG_ERROR("Failed to open engine file %s.", path.c_str()); + return InferencerErrorCode::TENSORRT_ENGINE_ERROR; + } + out_stream.write(plan.data(), plan.size()); + if (out_stream.bad()) { + GXF_LOG_ERROR("Failed writing engine file %s.", path.c_str()); + return InferencerErrorCode::TENSORRT_ENGINE_ERROR; + } + out_stream.close(); + GXF_LOG_INFO("TensorRT engine serialized at %s", path.c_str()); + return InferencerErrorCode::SUCCESS; +} + +TensorRTInferencer::TensorRTInferencer(const TensorRTInferenceParams& params) + : m_logger(new TRTLogger()) + , m_maxBatchSize(params.maxBatchSize) + , m_inputLayers(params.inputLayerNames) + , m_outputLayers(params.outputLayerNames) + , m_cudaStream(0) + , m_inferenceEngine(nullptr) { + if (params.inferType == TRTInferenceType::TRT_ENGINE) { + std::ifstream trtModelFStream(params.engineFilePath, std::ios::binary); + const bool shouldRebuild = params.force_engine_update || !trtModelFStream.good(); + const bool canRebuild = params.onnxFilePath.size() != 0; + if (canRebuild && shouldRebuild) { + // Deletes engine plan file if exists for forced update + std::remove(params.engineFilePath.c_str()); + if (std::ifstream(params.engineFilePath).good()) { + GXF_LOG_ERROR("Failed to remove engine plan file %s for forced engine update.", + params.engineFilePath.c_str()); + } + GXF_LOG_INFO( + "Rebuilding CUDA engine %s%s. " + "Note: this process may take up to several minutes.", + params.force_engine_update ? " (forced by config)" : "", + params.engineFilePath.c_str()); + auto result = convertModelToEngine(params.dlaID, params.onnxFilePath.c_str(), + params.max_workspace_size, params.buildFlags, params.maxBatchSize); + if (result != InferencerErrorCode::SUCCESS) { + GXF_LOG_INFO("Failed to create engine plan for model %s.", + params.onnxFilePath.c_str()); + } + + // Tries to serializes the plan and proceeds anyway + if (SerializeEnginePlan(m_modelEngineStream, params.engineFilePath) != + InferencerErrorCode::SUCCESS) { + GXF_LOG_INFO( + "Engine plan serialization failed. Proceeds with in-memory" + "engine plan anyway."); + } + } else { + GXF_LOG_INFO("Using CUDA engine %s. ", params.engineFilePath.c_str()); + + trtModelFStream.seekg(0, trtModelFStream.end); + m_modelEngineStreamSize = trtModelFStream.tellg(); + m_modelEngineStream.resize(m_modelEngineStreamSize); + trtModelFStream.seekg(0, trtModelFStream.beg); + trtModelFStream.read(m_modelEngineStream.data(), m_modelEngineStreamSize); + trtModelFStream.close(); + } + + m_inferenceRuntime.reset(nvinfer1::createInferRuntime(*(m_logger.get()))); + if (params.dlaID != -1 && params.dlaID < m_inferenceRuntime->getNbDLACores()) { + m_inferenceRuntime->setDLACore(params.dlaID); + } + m_inferenceEngine = m_inferenceRuntime->deserializeCudaEngine(m_modelEngineStream.data(), + m_modelEngineStreamSize); + m_ownedInferenceEngine.reset(m_inferenceEngine); + m_inferenceContext.reset(m_inferenceEngine->createExecutionContext()); + m_inferenceContext->setOptimizationProfileAsync(0, m_cudaStream); + } else { + if (params.engine == nullptr) { + throw ErrorCode::INVALID_ARGUMENT; + } + m_inferenceEngine = params.engine; + m_inferenceContext.reset(m_inferenceEngine->createExecutionContext()); + } + + if (m_inferenceEngine == nullptr || m_inferenceContext == nullptr) { + throw ErrorCode::INVALID_ARGUMENT; + } + + m_hasImplicitBatch = m_inferenceEngine->hasImplicitBatchDimension(); + m_bindingsCount = m_inferenceEngine->getNbBindings(); + if (!m_hasImplicitBatch) { + for (size_t i = 0; i < m_bindingsCount; i++) { + if (m_inferenceEngine->bindingIsInput(i)) { + nvinfer1::Dims dims_i(m_inferenceEngine->getBindingDimensions(i)); + nvinfer1::Dims4 inputDims{1, dims_i.d[1], dims_i.d[2], dims_i.d[3]}; + m_inferenceContext->setBindingDimensions(i, inputDims); + } + } + } + std::error_code err; + err = ParseTRTModel(); + if (err != make_error_code(ErrorCode::SUCCESS)) { + throw err; + } + m_buffers.resize(m_bindingsCount); +} + +// Set input layer tensor +std::error_code TensorRTInferencer::setInput(const TensorBase& trtInputBuffer, + std::string inputLayerName) { + if (m_modelInfo.inputLayers.find(inputLayerName) == m_modelInfo.inputLayers.end()) { + return ErrorCode::INVALID_ARGUMENT; + } + LayerInfo layer = m_modelInfo.inputLayers[inputLayerName]; + m_buffers[layer.index] = trtInputBuffer.getData(); + return ErrorCode::SUCCESS; +} + +// Sets output layer tensor +std::error_code TensorRTInferencer::setOutput(TensorBase& trtOutputBuffer, + std::string outputLayerName) { + if (m_modelInfo.outputLayers.find(outputLayerName) == m_modelInfo.outputLayers.end()) { + return ErrorCode::INVALID_ARGUMENT; + } + LayerInfo layer = m_modelInfo.outputLayers[outputLayerName]; + m_buffers[layer.index] = trtOutputBuffer.getData(); + return ErrorCode::SUCCESS; +} + +// Get the model metadata parsed based on the model file +// This would be done in initialize call itself. User can access the modelMetaData +// created using this API. +ModelMetaData TensorRTInferencer::getModelMetaData() const { + return m_modelInfo; +} + +std::error_code TensorRTInferencer::infer(size_t batchSize) { + bool err = true; + if (!m_hasImplicitBatch) { + size_t bindingsCount = m_inferenceEngine->getNbBindings(); + for (size_t i = 0; i < bindingsCount; i++) { + if (m_inferenceEngine->bindingIsInput(i)) { + nvinfer1::Dims dims_i(m_inferenceEngine->getBindingDimensions(i)); + nvinfer1::Dims4 inputDims{static_cast(batchSize), dims_i.d[1], + dims_i.d[2], dims_i.d[3]}; + m_inferenceContext->setBindingDimensions(i, inputDims); + } + } + err = m_inferenceContext->enqueueV2(&m_buffers[0], m_cudaStream, nullptr); + } else { + err = m_inferenceContext->enqueue(m_maxBatchSize, &m_buffers[0], m_cudaStream, nullptr); + } + if (!err) { + return InferencerErrorCode::TENSORRT_INFERENCE_ERROR; + } + return ErrorCode::SUCCESS; +} + +// Applicable only for Native TRT +std::error_code TensorRTInferencer::setCudaStream(cudaStream_t cudaStream) { + m_cudaStream = cudaStream; + return ErrorCode::SUCCESS; +} + +std::error_code TensorRTInferencer::unregister(std::string layerName) { + size_t index; + if (m_modelInfo.outputLayers.find(layerName) != m_modelInfo.outputLayers.end()) { + index = m_modelInfo.outputLayers[layerName].index; + } else if (m_modelInfo.inputLayers.find(layerName) != m_modelInfo.inputLayers.end()) { + index = m_modelInfo.inputLayers[layerName].index; + } else { + return ErrorCode::INVALID_ARGUMENT; + } + m_buffers[index] = nullptr; + return ErrorCode::SUCCESS; +} + +std::error_code TensorRTInferencer::unregister() { + for (size_t i = 0; i < m_buffers.size(); i++) { + m_buffers[i] = nullptr; + } + return ErrorCode::SUCCESS; +} + +TensorRTInferencer::~TensorRTInferencer() { + m_buffers.clear(); +} + +} // namespace inferencer +} // namespace cvcore diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/inferencer/tensorrt/TensorRTInferencer.h b/isaac_ros_bi3d/gxf/gems/dnn_inferencer/inferencer/TensorRTInferencer.h similarity index 54% rename from isaac_ros_bi3d/gxf/bi3d/cvcore/src/inferencer/tensorrt/TensorRTInferencer.h rename to isaac_ros_bi3d/gxf/gems/dnn_inferencer/inferencer/TensorRTInferencer.h index b2a6535..21255ff 100644 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/inferencer/tensorrt/TensorRTInferencer.h +++ b/isaac_ros_bi3d/gxf/gems/dnn_inferencer/inferencer/TensorRTInferencer.h @@ -15,46 +15,58 @@ // // SPDX-License-Identifier: Apache-2.0 -#ifndef TENSORRT_INFERENCER_H -#define TENSORRT_INFERENCER_H +#pragma once #include +#include +#include +#include #include -#include "cv/inferencer/Errors.h" -#include "cv/inferencer/IInferenceBackend.h" -#include "cv/inferencer/Inferencer.h" +#include +#include "Errors.h" +#include "IInferenceBackend.h" +#include "Inferencer.h" -namespace cvcore { namespace inferencer { +namespace cvcore { +namespace inferencer { -class TensorRTInferencer : public IInferenceBackendClient -{ -public: - TensorRTInferencer(const TensorRTInferenceParams ¶ms); +using TensorBase = cvcore::tensor_ops::TensorBase; +class TensorRTInferencer : public IInferenceBackendClient { + public: + TensorRTInferencer(const TensorRTInferenceParams& params); // Set input layer tensor - virtual std::error_code setInput(const cvcore::TensorBase &trtInputBuffer, std::string inputLayerName) override; + std::error_code setInput(const TensorBase& trtInputBuffer, + std::string inputLayerName) override; // Sets output layer tensor - virtual std::error_code setOutput(cvcore::TensorBase &trtOutputBuffer, std::string outputLayerName) override; + std::error_code setOutput(TensorBase& trtOutputBuffer, + std::string outputLayerName) override; // Get the model metadata parsed based on the model file - // This would be done in initialize call itself. User can access the modelMetaData created using this API. - virtual ModelMetaData getModelMetaData() const override; + // This would be done in initialize call itself. User can access the + // modelMetaData created using this API. + ModelMetaData getModelMetaData() const override; + + // Convert onnx mode to engine file + std::error_code convertModelToEngine(int32_t dla_core, + const char* model_file, int64_t max_workspace_size, int32_t buildFlags, + std::size_t max_batch_size); // TensorRT will use infer and TensorRT would use enqueueV2 - virtual std::error_code infer(size_t batchSize = 1) override; + std::error_code infer(size_t batchSize = 1) override; // Applicable only for Native TRT - virtual std::error_code setCudaStream(cudaStream_t) override; // Only in TRT + std::error_code setCudaStream(cudaStream_t) override; // Unregister shared memory for layer - virtual std::error_code unregister(std::string layerName) override; + std::error_code unregister(std::string layerName) override; // Unregister all shared memory - virtual std::error_code unregister() override; + std::error_code unregister() override; -private: + private: ~TensorRTInferencer(); std::unique_ptr m_logger; std::unique_ptr m_inferenceRuntime; @@ -62,17 +74,19 @@ class TensorRTInferencer : public IInferenceBackendClient std::vector m_inputLayers; std::vector m_outputLayers; cudaStream_t m_cudaStream; - nvinfer1::ICudaEngine *m_inferenceEngine; + nvinfer1::ICudaEngine* m_inferenceEngine; std::unique_ptr m_ownedInferenceEngine; std::unique_ptr m_inferenceContext; size_t m_bindingsCount; ModelMetaData m_modelInfo; - std::vector m_buffers; + std::vector m_buffers; bool m_hasImplicitBatch; + std::vector m_modelEngineStream; + size_t m_modelEngineStreamSize = 0; std::error_code ParseTRTModel(); - std::error_code getLayerInfo(LayerInfo &layer, std::string layerName); + std::error_code getLayerInfo(LayerInfo& layer, std::string layerName); }; -}} // namespace cvcore::inferencer -#endif +} // namespace inferencer +} // namespace cvcore diff --git a/isaac_ros_bi3d/gxf/gems/dnn_inferencer/inferencer/TensorRTUtils.cpp b/isaac_ros_bi3d/gxf/gems/dnn_inferencer/inferencer/TensorRTUtils.cpp new file mode 100644 index 0000000..d370049 --- /dev/null +++ b/isaac_ros_bi3d/gxf/gems/dnn_inferencer/inferencer/TensorRTUtils.cpp @@ -0,0 +1,57 @@ +// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES +// Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +#include "gems/dnn_inferencer/inferencer/TensorRTUtils.h" +#include + +namespace cvcore { +namespace inferencer { +using TensorLayout = cvcore::tensor_ops::TensorLayout; +using ChannelType = cvcore::tensor_ops::ChannelType; +using ChannelCount = cvcore::tensor_ops::ChannelCount; +using ErrorCode = cvcore::tensor_ops::ErrorCode; + +std::error_code getCVCoreChannelTypeFromTensorRT(ChannelType& channelType, + nvinfer1::DataType dtype) { + if (dtype == nvinfer1::DataType::kINT8) { + channelType = ChannelType::U8; + } else if (dtype == nvinfer1::DataType::kHALF) { + channelType = ChannelType::F16; + } else if (dtype == nvinfer1::DataType::kFLOAT) { + channelType = ChannelType::F32; + } else { + return ErrorCode::INVALID_OPERATION; + } + + return ErrorCode::SUCCESS; +} + +std::error_code getCVCoreChannelLayoutFromTensorRT(TensorLayout& channelLayout, + nvinfer1::TensorFormat tensorFormat) { + if (tensorFormat == nvinfer1::TensorFormat::kLINEAR) { + channelLayout = TensorLayout::NCHW; + } else if (tensorFormat == nvinfer1::TensorFormat::kHWC) { + channelLayout = TensorLayout::HWC; + } else { + return ErrorCode::INVALID_OPERATION; + } + + return ErrorCode::SUCCESS; +} + +} // namespace inferencer +} // namespace cvcore diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/inferencer/tensorrt/TensorRTUtils.h b/isaac_ros_bi3d/gxf/gems/dnn_inferencer/inferencer/TensorRTUtils.h similarity index 69% rename from isaac_ros_bi3d/gxf/bi3d/cvcore/src/inferencer/tensorrt/TensorRTUtils.h rename to isaac_ros_bi3d/gxf/gems/dnn_inferencer/inferencer/TensorRTUtils.h index 290ea70..e4139f5 100644 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/inferencer/tensorrt/TensorRTUtils.h +++ b/isaac_ros_bi3d/gxf/gems/dnn_inferencer/inferencer/TensorRTUtils.h @@ -14,14 +14,14 @@ // limitations under the License. // // SPDX-License-Identifier: Apache-2.0 -#ifndef TENSORRT_UTILS_H -#define TENSORRT_UTILS_H +#pragma once #include "NvInferRuntime.h" -#include "cv/core/Tensor.h" -#include "cv/inferencer/Errors.h" +#include "extensions/tensorops/core/Tensor.h" +#include "Errors.h" -namespace cvcore { namespace inferencer { +namespace cvcore { +namespace inferencer { /* * Maps tensorrt datatype to cvcore Channel type. @@ -29,7 +29,9 @@ namespace cvcore { namespace inferencer { * @param dtype tensorrt datatype * return error code */ -std::error_code getCVCoreChannelTypeFromTensorRT(cvcore::ChannelType &channelType, nvinfer1::DataType dtype); +std::error_code getCVCoreChannelTypeFromTensorRT( + cvcore::tensor_ops::ChannelType& channelType, + nvinfer1::DataType dtype); /* * Maps tensorrt datatype to cvcore Channel type. @@ -37,9 +39,9 @@ std::error_code getCVCoreChannelTypeFromTensorRT(cvcore::ChannelType &channelTyp * @param dtype tensorrt layout * return error code */ -std::error_code getCVCoreChannelLayoutFromTensorRT(cvcore::TensorLayout &channelLayout, - nvinfer1::TensorFormat tensorFormat); +std::error_code getCVCoreChannelLayoutFromTensorRT( + cvcore::tensor_ops::TensorLayout& channelLayout, + nvinfer1::TensorFormat tensorFormat); -}} // namespace cvcore::inferencer - -#endif +} // namespace inferencer +} // namespace cvcore diff --git a/isaac_ros_bi3d/gxf/gems/dnn_inferencer/inferencer/TritionUtils.cpp b/isaac_ros_bi3d/gxf/gems/dnn_inferencer/inferencer/TritionUtils.cpp new file mode 100644 index 0000000..1c27d6d --- /dev/null +++ b/isaac_ros_bi3d/gxf/gems/dnn_inferencer/inferencer/TritionUtils.cpp @@ -0,0 +1,64 @@ +// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES +// Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 +#ifdef ENABLE_TRITON + +#include +#include +#include "gems/dnn_inferencer/inferencer/TritonUtils.h" + +namespace cvcore { +namespace inferencer { +using ChannelType = cvcore::tensor_ops::ChannelType; +bool getCVCoreChannelType(ChannelType& channelType, std::string dtype) { + if (dtype.compare("UINT8") == 0) { + channelType = ChannelType::U8; + } else if (dtype.compare("UINT16") == 0) { + channelType = ChannelType::U16; + } else if (dtype.compare("FP16") == 0) { + channelType = ChannelType::F16; + } else if (dtype.compare("FP32") == 0) { + channelType = ChannelType::F32; + } else if (dtype.compare("FP64") == 0) { + channelType = ChannelType::F64; + } else { + return false; + } + + return true; +} + +bool getTritonChannelType(std::string& dtype, ChannelType channelType) { + if (channelType == ChannelType::U8) { + dtype = "UINT8"; + } else if (channelType == ChannelType::U16) { + dtype = "UINT16"; + } else if (channelType == ChannelType::F16) { + dtype = "FP16"; + } else if (channelType == ChannelType::F32) { + dtype = "FP32"; + } else if (channelType == ChannelType::F64) { + dtype = "FP64"; + } else { + return false; + } + + return true; +} + +} // namespace inferencer +} // namespace cvcore +#endif diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/inferencer/triton/TritonGrpcInferencer.cpp b/isaac_ros_bi3d/gxf/gems/dnn_inferencer/inferencer/TritonGrpcInferencer.cpp similarity index 74% rename from isaac_ros_bi3d/gxf/bi3d/cvcore/src/inferencer/triton/TritonGrpcInferencer.cpp rename to isaac_ros_bi3d/gxf/gems/dnn_inferencer/inferencer/TritonGrpcInferencer.cpp index f30e8c1..2e4b3ad 100644 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/inferencer/triton/TritonGrpcInferencer.cpp +++ b/isaac_ros_bi3d/gxf/gems/dnn_inferencer/inferencer/TritonGrpcInferencer.cpp @@ -15,29 +15,32 @@ // // SPDX-License-Identifier: Apache-2.0 #ifdef ENABLE_TRITON -#include "TritonGrpcInferencer.h" +#include "gems/dnn_inferencer/inferencer/TritonGrpcInferencer.h" #include #include +#include +#include #include "TritonUtils.h" -#include "cv/inferencer/Errors.h" -#include "cv/inferencer/IInferenceBackend.h" -#include "cv/inferencer/Inferencer.h" -namespace cvcore { namespace inferencer { +#include "gems/dnn_inferencer/inferencer/Errors.h" +#include "gems/dnn_inferencer/inferencer/IInferenceBackend.h" +#include "gems/dnn_inferencer/inferencer/Inferencer.h" + +namespace cvcore { +namespace inferencer { namespace tc = triton::client; +using ChannelType = cvcore::tensor_ops::ChannelType; namespace { -size_t getDataSize(const std::vector &shape, cvcore::ChannelType dataType) -{ +size_t getDataSize(const std::vector& shape, ChannelType dataType) { size_t layerShape = 1; for (size_t k = 0; k < shape.size(); k++) layerShape *= shape[k] <= 0 ? 1 : shape[k]; return layerShape * GetChannelSize(dataType); } -} // namespace +} // namespace -std::error_code TritonGrpcInferencer::ParseGrpcModel() -{ +std::error_code TritonGrpcInferencer::ParseGrpcModel() { inference::ModelMetadataResponse tritonModelMetadata; inference::ModelConfigResponse modelConfig; @@ -48,43 +51,39 @@ std::error_code TritonGrpcInferencer::ParseGrpcModel() err = client->ModelConfig(&modelConfig, modelName, modelVersion); modelInfo.maxBatchSize = modelConfig.config().max_batch_size(); bool inputBatchDim = modelInfo.maxBatchSize > 0; - for (int i = 0; i < tritonModelMetadata.inputs().size(); i++) - { + for (int i = 0; i < tritonModelMetadata.inputs().size(); i++) { LayerInfo layer; layer.name = tritonModelMetadata.inputs(i).name(); layer.index = i; - bool parseStatus = getCVCoreChannelType(layer.dataType, tritonModelMetadata.inputs(i).datatype()); - if (!parseStatus) - { + bool parseStatus = getCVCoreChannelType(layer.dataType, + tritonModelMetadata.inputs(i).datatype()); + if (!parseStatus) { return ErrorCode::INVALID_OPERATION; } size_t cnt = modelInfo.maxBatchSize == 0 ? 0 : 1; if (modelInfo.maxBatchSize != 0) layer.shape.push_back(modelInfo.maxBatchSize); - for (; cnt < tritonModelMetadata.inputs(i).shape().size(); cnt++) - { + for (; cnt < tritonModelMetadata.inputs(i).shape().size(); cnt++) { layer.shape.push_back(tritonModelMetadata.inputs(i).shape(cnt)); } layer.layerSize = getDataSize(layer.shape, layer.dataType); modelInfo.inputLayers[layer.name] = layer; } - for (int i = 0; i < tritonModelMetadata.outputs().size(); i++) - { + for (int i = 0; i < tritonModelMetadata.outputs().size(); i++) { LayerInfo layer; layer.name = tritonModelMetadata.outputs(i).name(); layer.index = i; - bool parseStatus = getCVCoreChannelType(layer.dataType, tritonModelMetadata.inputs(i).datatype()); - if (!parseStatus) - { + bool parseStatus = getCVCoreChannelType(layer.dataType, + tritonModelMetadata.inputs(i).datatype()); + if (!parseStatus) { return ErrorCode::INVALID_OPERATION; } layer.layout = TensorLayout::NHWC; size_t cnt = modelInfo.maxBatchSize == 0 ? 0 : 1; if (modelInfo.maxBatchSize != 0) layer.shape.push_back(modelInfo.maxBatchSize); - for (; cnt < tritonModelMetadata.outputs(i).shape().size(); cnt++) - { + for (; cnt < tritonModelMetadata.outputs(i).shape().size(); cnt++) { layer.shape.push_back(tritonModelMetadata.outputs(i).shape(cnt)); } modelInfo.outputLayers[layer.name] = layer; @@ -93,28 +92,24 @@ std::error_code TritonGrpcInferencer::ParseGrpcModel() return ErrorCode::SUCCESS; } -TritonGrpcInferencer::TritonGrpcInferencer(const TritonRemoteInferenceParams ¶ms) +TritonGrpcInferencer::TritonGrpcInferencer(const TritonRemoteInferenceParams& params) : modelVersion(params.modelVersion) - , modelName(params.modelName) -{ - - tc::Error err = tc::InferenceServerGrpcClient::Create(&client, params.serverUrl, params.verbose); + , modelName(params.modelName) { + tc::Error err = tc::InferenceServerGrpcClient::Create(&client, params.serverUrl, + params.verbose); - if (!err.IsOk()) - { + if (!err.IsOk()) { throw make_error_code(InferencerErrorCode::TRITON_SERVER_NOT_READY); } // Unregistering all shared memory regions for a clean // start. err = client->UnregisterSystemSharedMemory(); - if (!err.IsOk()) - { + if (!err.IsOk()) { throw make_error_code(InferencerErrorCode::TRITON_CUDA_SHARED_MEMORY_ERROR); } err = client->UnregisterCudaSharedMemory(); - if (!err.IsOk()) - { + if (!err.IsOk()) { throw make_error_code(InferencerErrorCode::TRITON_CUDA_SHARED_MEMORY_ERROR); } @@ -127,46 +122,40 @@ TritonGrpcInferencer::TritonGrpcInferencer(const TritonRemoteInferenceParams &pa outputRequests.resize(modelInfo.outputLayers.size()); outputMap.resize(modelInfo.outputLayers.size()); outputMapHistory.resize(modelInfo.outputLayers.size()); - for (auto &it : modelInfo.inputLayers) - { + for (auto &it : modelInfo.inputLayers) { tc::InferInput *inferInputVal; std::string tritonDataType; bool parseStatus = getTritonChannelType(tritonDataType, it.second.dataType); - if (!parseStatus) - { + if (!parseStatus) { throw make_error_code(InferencerErrorCode::TRITON_REGISTER_LAYER_ERROR); } - err = tc::InferInput::Create(&inferInputVal, it.second.name, it.second.shape, tritonDataType); - if (!err.IsOk()) - { + err = tc::InferInput::Create(&inferInputVal, it.second.name, it.second.shape, + tritonDataType); + if (!err.IsOk()) { throw make_error_code(InferencerErrorCode::TRITON_REGISTER_LAYER_ERROR); } inputRequests[it.second.index].reset(inferInputVal); } - for (auto &it : modelInfo.outputLayers) - { + for (auto &it : modelInfo.outputLayers) { tc::InferRequestedOutput *output; err = tc::InferRequestedOutput::Create(&output, it.second.name); - if (!err.IsOk()) - { + if (!err.IsOk()) { throw make_error_code(InferencerErrorCode::TRITON_REGISTER_LAYER_ERROR); } outputRequests[it.second.index].reset(output); } } -cudaError_t CreateCUDAIPCHandle(cudaIpcMemHandle_t *cuda_handle, void *input_d_ptr, int deviceId = 0) -{ +cudaError_t CreateCUDAIPCHandle(cudaIpcMemHandle_t* cuda_handle, void* input_d_ptr, + int deviceId = 0) { // Set the GPU device to the desired GPU cudaError_t err; err = cudaSetDevice(deviceId); - if (err != cudaSuccess) - { + if (err != cudaSuccess) { return err; } err = cudaIpcGetMemHandle(cuda_handle, input_d_ptr); - if (err != cudaSuccess) - { + if (err != cudaSuccess) { return err; } return cudaSuccess; @@ -174,43 +163,40 @@ cudaError_t CreateCUDAIPCHandle(cudaIpcMemHandle_t *cuda_handle, void *input_d_p } // Set input layer tensor -std::error_code TritonGrpcInferencer::setInput(const cvcore::TensorBase &trtInputBuffer, std::string inputLayerName) -{ - if (trtInputBuffer.isCPU()) - { +std::error_code TritonGrpcInferencer::setInput(const cvcore::TensorBase& trtInputBuffer, + std::string inputLayerName) { + if (trtInputBuffer.isCPU()) { return ErrorCode::INVALID_ARGUMENT; } size_t index = modelInfo.inputLayers[inputLayerName].index; - if (inputMapHistory[index] != (void *)trtInputBuffer.getData()) - { + if (inputMapHistory[index] != reinterpret_cast(trtInputBuffer.getData())) { inputMapHistory[index] = trtInputBuffer.getData(); unregister(inputLayerName); cudaIpcMemHandle_t input_cuda_handle; - cudaError_t cudaStatus = CreateCUDAIPCHandle(&input_cuda_handle, (void *)trtInputBuffer.getData()); - if (cudaStatus != cudaSuccess) - { + cudaError_t cudaStatus = CreateCUDAIPCHandle(&input_cuda_handle, + reinterpret_cast(trtInputBuffer.getData())); + if (cudaStatus != cudaSuccess) { return make_error_code(InferencerErrorCode::TRITON_CUDA_SHARED_MEMORY_ERROR); } tc::Error err; err = client->RegisterCudaSharedMemory(inputLayerName.c_str(), input_cuda_handle, 0, trtInputBuffer.getDataSize()); - if (!err.IsOk()) - { + if (!err.IsOk()) { return make_error_code(InferencerErrorCode::TRITON_CUDA_SHARED_MEMORY_ERROR); } size_t index = modelInfo.inputLayers[inputLayerName].index; - err = inputRequests[index]->SetSharedMemory(inputLayerName.c_str(), trtInputBuffer.getDataSize(), 0); + err = inputRequests[index]->SetSharedMemory(inputLayerName.c_str(), + trtInputBuffer.getDataSize(), 0); inputMap[index] = inputRequests[index].get(); - if (!err.IsOk()) - { + if (!err.IsOk()) { return make_error_code(InferencerErrorCode::TRITON_CUDA_SHARED_MEMORY_ERROR); - err = inputRequests[index]->SetSharedMemory(inputLayerName.c_str(), trtInputBuffer.getDataSize(), 0); + err = inputRequests[index]->SetSharedMemory(inputLayerName.c_str(), + trtInputBuffer.getDataSize(), 0); inputMap[index] = inputRequests[index].get(); - if (!err.IsOk()) - { + if (!err.IsOk()) { return make_error_code(InferencerErrorCode::TRITON_CUDA_SHARED_MEMORY_ERROR); } } @@ -219,32 +205,29 @@ std::error_code TritonGrpcInferencer::setInput(const cvcore::TensorBase &trtInpu } // Sets output layer tensor -std::error_code TritonGrpcInferencer::setOutput(cvcore::TensorBase &trtOutputBuffer, std::string outputLayerName) -{ - if (trtOutputBuffer.isCPU()) - { +std::error_code TritonGrpcInferencer::setOutput(cvcore::TensorBase& trtOutputBuffer, + std::string outputLayerName) { + if (trtOutputBuffer.isCPU()) { return ErrorCode::INVALID_ARGUMENT; } size_t index = modelInfo.outputLayers[outputLayerName].index; - if (outputMapHistory[index] != (void *)trtOutputBuffer.getData()) - { + if (outputMapHistory[index] != reinterpret_cast(trtOutputBuffer.getData())) { outputMapHistory[index] = trtOutputBuffer.getData(); unregister(outputLayerName); cudaIpcMemHandle_t outputCudaHandle; - CreateCUDAIPCHandle(&outputCudaHandle, (void *)trtOutputBuffer.getData()); + CreateCUDAIPCHandle(&outputCudaHandle, + reinterpret_cast(trtOutputBuffer.getData())); tc::Error err; - err = client->RegisterCudaSharedMemory(outputLayerName.c_str(), outputCudaHandle, 0 /* deviceId */, + err = client->RegisterCudaSharedMemory(outputLayerName.c_str(), outputCudaHandle, 0, trtOutputBuffer.getDataSize()); - if (!err.IsOk()) - { + if (!err.IsOk()) { return make_error_code(InferencerErrorCode::TRITON_CUDA_SHARED_MEMORY_ERROR); } - err = outputRequests[index]->SetSharedMemory(outputLayerName.c_str(), trtOutputBuffer.getDataSize(), - 0 /* offset */); - if (!err.IsOk()) - { + err = outputRequests[index]->SetSharedMemory(outputLayerName.c_str(), + trtOutputBuffer.getDataSize(), 0); + if (!err.IsOk()) { return make_error_code(InferencerErrorCode::TRITON_CUDA_SHARED_MEMORY_ERROR); } outputMap[index] = outputRequests[index].get(); @@ -253,46 +236,40 @@ std::error_code TritonGrpcInferencer::setOutput(cvcore::TensorBase &trtOutputBuf } // Get the model metadata parsed based on the model file -// This would be done in initialize call itself. User can access the modelMetaData created using this API. -ModelMetaData TritonGrpcInferencer::getModelMetaData() const -{ +// This would be done in initialize call itself. User can access the modelMetaData created +// using this API. +ModelMetaData TritonGrpcInferencer::getModelMetaData() const { return modelInfo; } // Triton will use infer and TensorRT would use enqueueV2 -std::error_code TritonGrpcInferencer::infer(size_t batchSize) -{ +std::error_code TritonGrpcInferencer::infer(size_t batchSize) { tc::InferResult *results; tc::Headers httpHeaders; tc::InferOptions options(modelInfo.modelName); options.model_version_ = modelInfo.modelVersion; - for (auto &inputLayer : modelInfo.inputLayers) - { + for (auto &inputLayer : modelInfo.inputLayers) { LayerInfo inputLayerInfo = inputLayer.second; size_t index = inputLayerInfo.index; tc::Error err; - err = - inputRequests[index]->SetSharedMemory(inputLayerInfo.name.c_str(), inputLayerInfo.layerSize * batchSize, 0); - if (!err.IsOk()) - { + err = inputRequests[index]->SetSharedMemory(inputLayerInfo.name.c_str(), + inputLayerInfo.layerSize * batchSize, 0); + if (!err.IsOk()) { return make_error_code(InferencerErrorCode::TRITON_CUDA_SHARED_MEMORY_ERROR); } } - for (auto &outputLayer : modelInfo.outputLayers) - { + for (auto &outputLayer : modelInfo.outputLayers) { LayerInfo outputLayerInfo = outputLayer.second; size_t index = outputLayerInfo.index; tc::Error err; err = outputRequests[index]->SetSharedMemory(outputLayerInfo.name.c_str(), - outputLayerInfo.layerSize * batchSize, 0); - if (!err.IsOk()) - { + outputLayerInfo.layerSize * batchSize, 0); + if (!err.IsOk()) { return make_error_code(InferencerErrorCode::TRITON_CUDA_SHARED_MEMORY_ERROR); } } tc::Error err = client->Infer(&results, options, inputMap, outputMap, httpHeaders); - if (!err.IsOk()) - { + if (!err.IsOk()) { return make_error_code(InferencerErrorCode::TRITON_INFERENCE_ERROR); } @@ -300,43 +277,37 @@ std::error_code TritonGrpcInferencer::infer(size_t batchSize) } // Applicable only for Native TRT -std::error_code TritonGrpcInferencer::setCudaStream(cudaStream_t) // Only in TRT -{ +std::error_code TritonGrpcInferencer::setCudaStream(cudaStream_t) { return ErrorCode::INVALID_OPERATION; } -std::error_code TritonGrpcInferencer::unregister(std::string layerName) -{ +std::error_code TritonGrpcInferencer::unregister(std::string layerName) { tc::Error err; inference::CudaSharedMemoryStatusResponse status; err = client->CudaSharedMemoryStatus(&status); - if (!err.IsOk()) - { + if (!err.IsOk()) { return make_error_code(InferencerErrorCode::TRITON_CUDA_SHARED_MEMORY_ERROR); } err = client->UnregisterCudaSharedMemory(layerName.c_str()); - if (!err.IsOk()) - { + if (!err.IsOk()) { return make_error_code(InferencerErrorCode::TRITON_CUDA_SHARED_MEMORY_ERROR); } return ErrorCode::SUCCESS; } -std::error_code TritonGrpcInferencer::unregister() -{ +std::error_code TritonGrpcInferencer::unregister() { tc::Error err; inference::CudaSharedMemoryStatusResponse status; err = client->CudaSharedMemoryStatus(&status); - if (!err.IsOk()) - { + if (!err.IsOk()) { return make_error_code(InferencerErrorCode::TRITON_CUDA_SHARED_MEMORY_ERROR); } err = client->UnregisterCudaSharedMemory(); - if (!err.IsOk()) - { + if (!err.IsOk()) { return make_error_code(InferencerErrorCode::TRITON_CUDA_SHARED_MEMORY_ERROR); } return ErrorCode::SUCCESS; } -}} // namespace cvcore::inferencer +} // namespace inferencer +} // namespace cvcore #endif diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/inferencer/triton/TritonGrpcInferencer.h b/isaac_ros_bi3d/gxf/gems/dnn_inferencer/inferencer/TritonGrpcInferencer.h similarity index 60% rename from isaac_ros_bi3d/gxf/bi3d/cvcore/src/inferencer/triton/TritonGrpcInferencer.h rename to isaac_ros_bi3d/gxf/gems/dnn_inferencer/inferencer/TritonGrpcInferencer.h index d02b3e2..2e74e99 100644 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/inferencer/triton/TritonGrpcInferencer.h +++ b/isaac_ros_bi3d/gxf/gems/dnn_inferencer/inferencer/TritonGrpcInferencer.h @@ -14,62 +14,65 @@ // limitations under the License. // // SPDX-License-Identifier: Apache-2.0 - +#pragma once #ifdef ENABLE_TRITON -#ifndef TRITONGRPC_INFERENCER_H -#define TRITONGRPC_INFERENCER_H #include #include - +#include +#include +#include #include "cv/inferencer/Errors.h" #include "cv/inferencer/IInferenceBackend.h" #include "cv/inferencer/Inferencer.h" -namespace cvcore { namespace inferencer { +namespace cvcore { +namespace inferencer { namespace tc = triton::client; -class TritonGrpcInferencer : public IInferenceBackendClient -{ -public: - TritonGrpcInferencer(const TritonRemoteInferenceParams ¶ms); +class TritonGrpcInferencer : public IInferenceBackendClient { + public: + TritonGrpcInferencer(const TritonRemoteInferenceParams& params); // Set input layer tensor - virtual std::error_code setInput(const cvcore::TensorBase &trtInputBuffer, std::string inputLayerName) override; + std::error_code setInput(const cvcore::TensorBase& trtInputBuffer, + std::string inputLayerName) override; // Sets output layer tensor - virtual std::error_code setOutput(cvcore::TensorBase &trtOutputBuffer, std::string outputLayerName) override; + std::error_code setOutput(cvcore::TensorBase& trtOutputBuffer, + std::string outputLayerName) override; // Get the model metadata parsed based on the model file - // This would be done in initialize call itself. User can access the modelMetaData created using this API. - virtual ModelMetaData getModelMetaData() const override; + // This would be done in initialize call itself. User can access the modelMetaData + // created using this API. + ModelMetaData getModelMetaData() const override; // Triton will use infer and TensorRT would use enqueueV2 - virtual std::error_code infer(size_t batchSize = 1) override; + std::error_code infer(size_t batchSize = 1) override; // Applicable only for Native TRT - virtual std::error_code setCudaStream(cudaStream_t) override; // Only in TRT + std::error_code setCudaStream(cudaStream_t) override; // Unregister shared memory for layer - virtual std::error_code unregister(std::string layerName) override; + std::error_code unregister(std::string layerName) override; // Unregister all shared memory - virtual std::error_code unregister() override; + std::error_code unregister() override; -private: + private: // Parse grpc model std::error_code ParseGrpcModel(); std::unique_ptr client; ModelMetaData modelInfo; std::vector> inputRequests; std::vector> outputRequests; - std::vector inputMap; - std::vector inputMapHistory; - std::vector outputMapHistory; - std::vector outputMap; + std::vector inputMap; + std::vector inputMapHistory; + std::vector outputMapHistory; + std::vector outputMap; std::string modelVersion, modelName; }; -}} // namespace cvcore::inferencer -#endif +} // namespace inferencer +} // namespace cvcore #endif diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/inferencer/triton/TritonUtils.h b/isaac_ros_bi3d/gxf/gems/dnn_inferencer/inferencer/TritonUtils.h similarity index 81% rename from isaac_ros_bi3d/gxf/bi3d/cvcore/src/inferencer/triton/TritonUtils.h rename to isaac_ros_bi3d/gxf/gems/dnn_inferencer/inferencer/TritonUtils.h index bb13553..9a3ee8d 100644 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/inferencer/triton/TritonUtils.h +++ b/isaac_ros_bi3d/gxf/gems/dnn_inferencer/inferencer/TritonUtils.h @@ -14,16 +14,17 @@ // limitations under the License. // // SPDX-License-Identifier: Apache-2.0 -#ifdef ENABLE_TRITON -#ifndef TRITON_UTILS_H -#define TRITON_UTILS_H +#pragma once +#ifdef ENABLE_TRITON #include +#include #include "cv/core/Tensor.h" #include "cv/inferencer/Errors.h" -namespace cvcore { namespace inferencer { +namespace cvcore { +namespace inferencer { /* * Maps triton datatype to cvcore Channel type. @@ -31,7 +32,7 @@ namespace cvcore { namespace inferencer { * @param dtype String representing triton datatype * return bool returns false if mapping was not successful. */ -bool getCVCoreChannelType(cvcore::ChannelType &channelType, std::string dtype); +bool getCVCoreChannelType(cvcore::ChannelType& channelType, std::string dtype); /* * Maps triton datatype to cvcore Channel type. @@ -39,9 +40,8 @@ bool getCVCoreChannelType(cvcore::ChannelType &channelType, std::string dtype); * @param channelType cvcore channel type. * return bool returns false if mapping was not successful. */ -bool getTritonChannelType(std::string &dtype, cvcore::ChannelType channelType); - -}} // namespace cvcore::inferencer -#endif // TRITON_UTILS_H +bool getTritonChannelType(std::string& dtype, cvcore::ChannelType channelType); -#endif // ENABLE_TRITON +} // namespace inferencer +} // namespace cvcore +#endif // ENABLE_TRITON diff --git a/isaac_ros_bi3d/gxf/gems/gxf_helpers/common_expected_macro.hpp b/isaac_ros_bi3d/gxf/gems/gxf_helpers/common_expected_macro.hpp new file mode 100644 index 0000000..2606c6e --- /dev/null +++ b/isaac_ros_bi3d/gxf/gems/gxf_helpers/common_expected_macro.hpp @@ -0,0 +1,305 @@ +// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES +// Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 +#pragma once + +#include +#include +#include + +#include "common/expected.hpp" + +// Concatenates its two arguments. +#define EXPECTED_MACRO_INTERNAL_CONCAT(a, b) EXPECTED_MACRO_INTERNAL_CONCAT_IMPL(a, b) +#define EXPECTED_MACRO_INTERNAL_CONCAT_IMPL(a, b) a##b + +// Converts its argument to a string at compile time. +#define EXPECTED_MACRO_INTERNAL_TO_STRING(x) EXPECTED_MACRO_INTERNAL_TO_STRING_IMPL(x) +#define EXPECTED_MACRO_INTERNAL_TO_STRING_IMPL(x) #x + +// Gets the current location in the source code in the format "file:line". +#define EXPECTED_MACRO_INTERNAL_FILE_LINE() __FILE__ ":" EXPECTED_MACRO_INTERNAL_TO_STRING(__LINE__) + +// Helper to support logging a default message and an optional custom message. +#define EXPECTED_MACRO_INTERNAL_LOG_IN_EXPECT_MACRO(expression_result, expression_string, ...) \ + ::nvidia::internal::LogHelper( \ + __FILE__, __LINE__, expression_result, expression_string, ##__VA_ARGS__); + +#define EXPECTED_MACRO_INTERNAL_CHECK_EXPRESSION_IS_EXPECTED_VOID_OR_RESULT(expression_result) \ + static_assert( \ + ::nvidia::internal::IsExpectedVoid::value || \ + std::is_enum_v, \ + EXPECTED_MACRO_INTERNAL_FILE_LINE() ": GXF_RETURN_IF_ERROR can only be used with " \ + "expressions that return Expected or enum. For expressions returning Expected " \ + "use GXF_UNWRAP_OR_RETURN instead."); + +#define EXPECTED_MACRO_INTERNAL_CHECK_EXPRESSION_IS_EXPECTED_T(expression_result) \ + static_assert(::nvidia::internal::IsExpectedT::value, \ + EXPECTED_MACRO_INTERNAL_FILE_LINE() ": GXF_UNWRAP_OR_RETURN can only be used with " \ + "expressions that return Expected. For expressions returning Expected or enum " \ + "use RETURN_IF_ERROR instead."); + +// Evaluates an expression that returns an Expected or result +// enum. If the returned type contains an error it returns the +// error. This macro can be used in functions returning both +// Expected and result enum. +// +// Per default the macro already creates an error message that +// includes the evaluated expression. If needed an optional string can +// be passed that will be appended to the default error message. It is +// also possible to use format specifiers to customize the string. +// +// It is also possible to pass the Severity used for logging as an +// additional argument. This is required if using a custom error +// message. +// +// Example: +// Expected DoSomething(); +// Expected DoAnotherThing(); +// +// Expected foo(){ +// GXF_RETURN_IF_ERROR(DoSomething()); +// GXF_RETURN_IF_ERROR(DoAnotherThing(), Severity::WARNING); +// GXF_RETURN_IF_ERROR(DoAnotherThing(), Severity::WARNING, "Custom error message."); +// } +#define RETURN_IF_ERROR(expression, ...) \ + do { \ + auto maybe_result = (expression); \ + EXPECTED_MACRO_INTERNAL_CHECK_EXPRESSION_IS_EXPECTED_VOID_OR_RESULT(maybe_result) \ + if (!::nvidia::internal::IsValid(maybe_result)) { \ + EXPECTED_MACRO_INTERNAL_LOG_IN_EXPECT_MACRO(maybe_result, #expression, ##__VA_ARGS__) \ + return ::nvidia::internal::ProxyFactory::FromExpectedVoidOrResult(maybe_result); \ + } \ + } while (0) + +// Evaluates an expression that returns an Expected. If the returned type +// contains an error it returns the error, else it unwraps the value contained in the Expected. +// This macro can be used in functions returning both Expected and result enum. +// +// Per default the macro already creates an error message that includes the evaluated expression. If +// needed an optional string can be passed that will be appended to the default error message. It is +// also possible to use format specifiers to customize the string. +// +// It is also possible to pass the Severity used for logging as an additional argument. +// +// Note that this macro uses expression-statements (i.e. the ({ }) surrounding the macro) which are +// a non-standard functionality. However they are present in almost all compilers. We currently only +// know of MSVC that does not support this. +// +// Example: +// Expected GetString(); +// Expected GetAnotherString(); +// +// Expected CountCombinedStringLength(){ +// const std::string str1 = GXF_UNWRAP_OR_RETURN(GetString()); +// std::string str2; +// str2 = GXF_UNWRAP_OR_RETURN(GetAnotherString(), "This should not fail. Str1 has value %s.", +// str1.c_str()); +// const std::string str3 = GXF_UNWRAP_OR_RETURN(GetAnotherString(), Severity::WARNING); +// const std::string str4 = GXF_UNWRAP_OR_RETURN(GetAnotherString(), Severity::WARNING, +// "Custom error message"); +// return str1.size() + str2.size() + str3.size() + str4.size(); +// } +#define UNWRAP_OR_RETURN(expression, ...) \ + ({ \ + auto maybe_result = (expression); \ + EXPECTED_MACRO_INTERNAL_CHECK_EXPRESSION_IS_EXPECTED_T(maybe_result) \ + if (!::nvidia::internal::IsValid(maybe_result)) { \ + EXPECTED_MACRO_INTERNAL_LOG_IN_EXPECT_MACRO(maybe_result, #expression, ##__VA_ARGS__) \ + return ::nvidia::internal::ProxyFactory::FromExpectedValue(maybe_result); \ + } \ + std::move(maybe_result.value()); \ + }) + +namespace nvidia { +// This struct has to be specialized for a given result enum. +template +struct ExpectedMacroConfig { + constexpr static Error DefaultSuccess(); + constexpr static Error DefaultError(); + static std::string Name(Error error) { return std::to_string(static_cast(error)); } +}; +} // namespace nvidia + +namespace nvidia::internal { + +constexpr Severity kDefaultSeverity = Severity::ERROR; + +template +struct IsExpectedVoid : public std::false_type {}; + +template +struct IsExpectedVoid> : public std::true_type {}; + +template +struct IsExpectedT : public std::false_type {}; + +template +struct IsExpectedT> : public std::false_type {}; + +template +struct IsExpectedT> : public std::true_type {}; + +// Returns true if the passed result is valid. +template +constexpr bool IsValid(Error result) { + return result == ExpectedMacroConfig::DefaultSuccess(); +} + +// Returns true if the passed expected is valid. +template +constexpr bool IsValid(const Expected& expected) { + return static_cast(expected); +} + +template +constexpr Error GetError(const Expected& expected) { + return expected.error(); +} + +template +constexpr Error GetError(const Error error) { + return error; +} + +template +class UnexpectedOrErrorProxy; + +class ProxyFactory { + public: + // Constructs the proxy from a Expected. The Expected has to be in an error state. We + // do not check this because the macro should have already done this check. We use static + // methods instead of constructors to explicitly disallow construction from certain types in + // different situations. + template + static UnexpectedOrErrorProxy FromExpectedValue(const Expected& expected) { + static_assert( + !std::is_same::value, + "This function should not be used with Expected, use " + "FromExpectedVoidOrResult() instead."); + return UnexpectedOrErrorProxy(expected); + } + + // Constructs the proxy from a Expected. The Expected has to be in an error state. + // We do not check this because the macro should have already done this check. This function + // needs to be overloaded with a function taking in a result enum. We use static methods instead + // of constructors to explicitly disallow construction from certain types in different + // situations. + template + static UnexpectedOrErrorProxy FromExpectedVoidOrResult( + const Expected expected) { + return UnexpectedOrErrorProxy(expected); + } + + // Constructs the proxy from a result enum. The result enum has to be in an error state. We do + // not check this because the macro should have already done this check. This function needs to + // be overloaded with a function taking in an Expected. We use static methods instead of + // constructors to explicitly disallow construction from certain types in different situations. + template + constexpr static UnexpectedOrErrorProxy FromExpectedVoidOrResult(Error error) { + return UnexpectedOrErrorProxy(error); + } +}; + +// A proxy class to abstract away the difference between a result enum and an Expected. +// This class defines casting operators s.t. it can implicitly cast to both result enum and +// Expected. Thus in a function that returns one of the afore mentioned types one can simply +// return this proxy and then it will implicitly cast to the appropriate return type. +template +class UnexpectedOrErrorProxy { + public: + // Casts the proxy to an error. Note that this cast is not allowed to be explicit. The + // OtherError has to be an enum (or enum class). + template >> + constexpr operator OtherError() const { + return castToError(); + } + + // Casts the proxy to an Expected. Note that this cast is not allowed to be explicit. + template + constexpr operator Expected() const { + return castToUnexpected(); + } + + private: + // Constructs the proxy from a Expected. Note that the Expected needs to contain an error. + // We do not check for this because we rely on the macro already having done that check. + template + constexpr explicit UnexpectedOrErrorProxy(const Expected& expected) + : error_(expected.error()) {} + + // Constructs the proxy from a result enum. The result enum needs to be in an error state. + // We do not check for this because we rely on the macro already having done that check. + constexpr explicit UnexpectedOrErrorProxy(Error error) : error_(error) {} + + // Casts the proxy to any error type. If the error type is not equal to the proxy's error type, + // a default error is used. + template + constexpr OtherError castToError() const { + static_assert(std::is_enum_v, "Can only cast to errors of type enum."); + if constexpr (std::is_same_v) { + return error_; + } else { + return ExpectedMacroConfig::DefaultError(); + } + } + + // Casts the proxy to an unexpected using any error type. If the error type is not equal to the + // proxy's error type, a default error is used. + template + constexpr Unexpected castToUnexpected() const { + return Unexpected(castToError()); + } + + Error error_; + static_assert(std::is_enum_v, "Error has to be an enum."); + + friend ProxyFactory; +}; + +// Helper function for the logging in the above macros. This version should be used when the user +// also specifies the logging severity. The variadic arguments can be used to do string +// interpolation in the custom_text variable. +template +void LogHelper( + const char* file, int line, const ExpressionResult& expression_result, + const std::string& expression_string, Severity severity, const std::string& custom_txt = "", + Args... args) { + const auto error = GetError(expression_result); + using Error = std::remove_const_t; + const std::string text = "Expression '" + expression_string + "' failed with error '" + + ExpectedMacroConfig::Name(error) + "'. " + custom_txt; + // GCC is not able to do format security validation when the string + // is coming from a variadic template, even if the string is + // originally a char* ignore this warning until a more recent GCC + // version fixes this behavior +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wformat-security" + ::nvidia::Log(file, line, severity, text.c_str(), &args...); +#pragma GCC diagnostic pop +} + +// Overload of the LogHelper above. This version does not take the severity as an argument and used +// the default severity instead. +template +void LogHelper( + const char* file, int line, const ExpressionResult& expression_result, + const std::string& expression_string, const std::string& custom_text = "", Args... args) { + LogHelper( + file, line, expression_result, expression_string, kDefaultSeverity, custom_text, &args...); +} + +} // namespace nvidia::internal diff --git a/isaac_ros_bi3d/gxf/gems/gxf_helpers/element_type.hpp b/isaac_ros_bi3d/gxf/gems/gxf_helpers/element_type.hpp new file mode 100644 index 0000000..7fab97e --- /dev/null +++ b/isaac_ros_bi3d/gxf/gems/gxf_helpers/element_type.hpp @@ -0,0 +1,55 @@ +// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES +// Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 +#pragma once + +#include "engine/core/tensor/element_type.hpp" +#include "gxf/std/tensor.hpp" + +namespace nvidia { +namespace isaac { + +// Helps determine the Isaac element type given the GXF primitive type +::nvidia::isaac::ElementType ToIsaacElementType(gxf::PrimitiveType gxf_type) { + switch (gxf_type) { + case gxf::PrimitiveType::kInt8: + return ::nvidia::isaac::ElementType::kInt8; + case gxf::PrimitiveType::kUnsigned8: + return ::nvidia::isaac::ElementType::kUInt8; + case gxf::PrimitiveType::kInt16: + return ::nvidia::isaac::ElementType::kInt16; + case gxf::PrimitiveType::kUnsigned16: + return ::nvidia::isaac::ElementType::kUInt16; + case gxf::PrimitiveType::kInt32: + return ::nvidia::isaac::ElementType::kInt32; + case gxf::PrimitiveType::kUnsigned32: + return ::nvidia::isaac::ElementType::kUInt32; + case gxf::PrimitiveType::kInt64: + return ::nvidia::isaac::ElementType::kInt64; + case gxf::PrimitiveType::kUnsigned64: + return ::nvidia::isaac::ElementType::kUInt64; + case gxf::PrimitiveType::kFloat32: + return ::nvidia::isaac::ElementType::kFloat32; + case gxf::PrimitiveType::kFloat64: + return ::nvidia::isaac::ElementType::kFloat64; + case gxf::PrimitiveType::kCustom: + default: + return ::nvidia::isaac::ElementType::kUnknown; + } +} + +} // namespace isaac +} // namespace nvidia diff --git a/isaac_ros_bi3d/gxf/bi3d/bi3d_postprocessor/bi3d_postprocess/bi3d_postprocessor.cu.hpp b/isaac_ros_bi3d/gxf/gems/gxf_helpers/expected_macro.hpp similarity index 52% rename from isaac_ros_bi3d/gxf/bi3d/bi3d_postprocessor/bi3d_postprocess/bi3d_postprocessor.cu.hpp rename to isaac_ros_bi3d/gxf/gems/gxf_helpers/expected_macro.hpp index 6c7de9c..43499b7 100644 --- a/isaac_ros_bi3d/gxf/bi3d/bi3d_postprocessor/bi3d_postprocess/bi3d_postprocessor.cu.hpp +++ b/isaac_ros_bi3d/gxf/gems/gxf_helpers/expected_macro.hpp @@ -14,25 +14,25 @@ // limitations under the License. // // SPDX-License-Identifier: Apache-2.0 -#ifndef NVIDIA_ISAAC_ROS_EXTENSIONS_BI3D_POSTPROCESSOR_CU_HPP_ -#define NVIDIA_ISAAC_ROS_EXTENSIONS_BI3D_POSTPROCESSOR_CU_HPP_ +#pragma once -#include -#include +#include -#include "cuda.h" -#include "cuda_runtime.h" +#include "gems/gxf_helpers/common_expected_macro.hpp" +#include "gxf/core/expected.hpp" -namespace nvidia -{ -namespace isaac_ros -{ +// This customizes the expected macro, s.t. it can be used with gxf_result_t. +namespace nvidia { +template <> +struct ExpectedMacroConfig { + constexpr static gxf_result_t DefaultSuccess() { return GXF_SUCCESS; } + constexpr static gxf_result_t DefaultError() { return GXF_FAILURE; } + static std::string Name(gxf_result_t result) { return GxfResultStr(result); } +}; -void cuda_postprocess( - const float * input, float * output, int disparity, int imageHeight, - int imageWidth); +// For back-compatibility we define an alias to the original name of the macro when it was gxf +// specific. +#define GXF_RETURN_IF_ERROR RETURN_IF_ERROR +#define GXF_UNWRAP_OR_RETURN UNWRAP_OR_RETURN -} // namespace isaac_ros } // namespace nvidia - -#endif // NVIDIA_ISAAC_ROS_EXTENSIONS_BI3D_POSTPROCESSOR_CU_HPP_ diff --git a/isaac_ros_bi3d/gxf/gems/gxf_helpers/tensor.hpp b/isaac_ros_bi3d/gxf/gems/gxf_helpers/tensor.hpp new file mode 100644 index 0000000..5ae069e --- /dev/null +++ b/isaac_ros_bi3d/gxf/gems/gxf_helpers/tensor.hpp @@ -0,0 +1,137 @@ +// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES +// Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 +#pragma once + +#include "engine/core/math/types.hpp" +#include "engine/core/tensor/tensor.hpp" +#include "gxf/core/expected.hpp" +#include "gxf/std/tensor.hpp" + +namespace nvidia { +namespace isaac { + +// Creates an Isaac tensor view of a GXF host tensor +template +gxf::Expected<::nvidia::isaac::CpuTensorView> ToIsaacCpuTensorView(gxf::Tensor& tensor) { + if ((tensor.storage_type() != gxf::MemoryStorageType::kHost) && + (tensor.storage_type() != gxf::MemoryStorageType::kSystem)) { + GXF_LOG_ERROR("Tensor does not have host or system storage type"); + return gxf::Unexpected{GXF_INVALID_DATA_FORMAT}; + } + if (tensor.rank() != N) { + GXF_LOG_ERROR("Tensor rank mismatch. Expected: %d, actual: %d", N, tensor.rank()); + return gxf::Unexpected{GXF_INVALID_DATA_FORMAT}; + } + ::nvidia::isaac::Vector dimensions; + for (int i = 0; i < N; i++) { + dimensions[i] = tensor.shape().dimension(i); + } + + auto maybe_ptr = tensor.data(); + if (!maybe_ptr) { + GXF_LOG_ERROR("Requested type for tensor pointer does not match tensor data type"); + return gxf::ForwardError(maybe_ptr); + } + + return ::nvidia::isaac::CreateCpuTensorViewFromData(maybe_ptr.value(), dimensions.prod(), + dimensions); +} + +// Creates an Isaac tensor const view of a GXF host tensor +template +gxf::Expected<::nvidia::isaac::CpuTensorConstView> +ToIsaacCpuTensorView(const gxf::Tensor& tensor) { + if ((tensor.storage_type() != gxf::MemoryStorageType::kHost) && + (tensor.storage_type() != gxf::MemoryStorageType::kSystem)) { + GXF_LOG_ERROR("Tensor does not have host or system storage type"); + return gxf::Unexpected{GXF_INVALID_DATA_FORMAT}; + } + if (tensor.rank() != N) { + GXF_LOG_ERROR("Tensor does not have expected rank %i", N); + return gxf::Unexpected{GXF_INVALID_DATA_FORMAT}; + } + ::nvidia::isaac::Vector dimensions; + for (int i = 0; i < N; i++) { + dimensions[i] = tensor.shape().dimension(i); + } + + auto maybe_ptr = tensor.data(); + if (!maybe_ptr) { + GXF_LOG_ERROR("Requested type for tensor pointer does not match tensor data type"); + return gxf::ForwardError(maybe_ptr); + } + + return ::nvidia::isaac::CreateCpuTensorConstViewFromData, N> ( + maybe_ptr.value(), dimensions.prod(), dimensions); +} + +// Creates an Isaac Cuda tensor view of a GXF Cuda tensor +template +gxf::Expected<::nvidia::isaac::GpuTensorView> ToIsaacGpuTensorView(gxf::Tensor& tensor) { + if (tensor.storage_type() != gxf::MemoryStorageType::kDevice) { + GXF_LOG_ERROR("Tensor does not have device storage type"); + return gxf::Unexpected{GXF_INVALID_DATA_FORMAT}; + } + if (tensor.rank() != N) { + GXF_LOG_ERROR("Tensor does not have expected rank %i", N); + return gxf::Unexpected{GXF_INVALID_DATA_FORMAT}; + } + ::nvidia::isaac::Vector dimensions; + for (int i = 0; i < N; i++) { + dimensions[i] = tensor.shape().dimension(i); + } + + auto maybe_ptr = tensor.data(); + if (!maybe_ptr) { + GXF_LOG_ERROR("Requested type for tensor pointer does not match tensor data type"); + return gxf::ForwardError(maybe_ptr); + } + + return ::nvidia::isaac::CreateGpuTensorViewFromData(maybe_ptr.value(), dimensions.prod(), + dimensions); +} + +// Creates an Isaac Cuda tensor const view of a GXF Cuda tensor +template +gxf::Expected<::nvidia::isaac::GpuTensorConstView> +ToIsaacGpuTensorView(const gxf::Tensor& tensor) { + if (tensor.storage_type() != gxf::MemoryStorageType::kDevice) { + GXF_LOG_ERROR("Tensor does not have device storage type"); + return gxf::Unexpected{GXF_INVALID_DATA_FORMAT}; + } + if (tensor.rank() != N) { + GXF_LOG_ERROR("Tensor does not have expected rank %i", N); + return gxf::Unexpected{GXF_INVALID_DATA_FORMAT}; + } + ::nvidia::isaac::Vector dimensions; + for (int i = 0; i < N; i++) { + dimensions[i] = tensor.shape().dimension(i); + } + + auto maybe_ptr = tensor.data(); + if (!maybe_ptr) { + GXF_LOG_ERROR("Requested type for tensor pointer does not match tensor data type"); + return gxf::ForwardError(maybe_ptr); + } + + return ::nvidia::isaac::CreateGpuTensorConstViewFromData(maybe_ptr.value(), + dimensions.prod(), + dimensions); +} + +} // namespace isaac +} // namespace nvidia diff --git a/isaac_ros_bi3d/gxf/gems/hash/hash_file.cpp b/isaac_ros_bi3d/gxf/gems/hash/hash_file.cpp new file mode 100644 index 0000000..3f7aead --- /dev/null +++ b/isaac_ros_bi3d/gxf/gems/hash/hash_file.cpp @@ -0,0 +1,71 @@ +// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES +// Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +#include +#include // NOLINT(build/include_order) + +#include "common/fixed_vector.hpp" +#include "common/span.hpp" +#include "gems/gxf_helpers/expected_macro.hpp" +#include "gems/hash/hash_file.hpp" + +namespace nvidia { +namespace isaac { + +namespace { + +constexpr size_t kBlockSize = 8096; + +} // namespace + +// helper to get the hash of a file +gxf::Expected hash_file(const char* path) { + std::ifstream file; + SHA256 hash; + FixedVector buffer; + + size_t bytes_remaining = std::filesystem::file_size(path); + + file.open(path, std::ios_base::in | std::ios_base::binary); + + if (!file.good()) { + return nvidia::gxf::Unexpected{GXF_FAILURE}; + } + + while (true) { + size_t current_bytes = std::min(kBlockSize, bytes_remaining); + + file.read(reinterpret_cast(buffer.data()), current_bytes); + + if (!file.good()) { + return nvidia::gxf::Unexpected{GXF_FAILURE}; + } + + GXF_RETURN_IF_ERROR(hash.hashData(Span(buffer.data(), current_bytes))); + + bytes_remaining -= current_bytes; + + if (bytes_remaining == 0) { + GXF_RETURN_IF_ERROR(hash.finalize()); + + return hash.toString(); + } + } +} + +} // namespace isaac +} // namespace nvidia diff --git a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/IImageWarp.cpp b/isaac_ros_bi3d/gxf/gems/hash/hash_file.hpp similarity index 68% rename from isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/IImageWarp.cpp rename to isaac_ros_bi3d/gxf/gems/hash/hash_file.hpp index c96b07e..72cd3ca 100644 --- a/isaac_ros_bi3d/gxf/bi3d/cvcore/src/tensor_ops/IImageWarp.cpp +++ b/isaac_ros_bi3d/gxf/gems/hash/hash_file.hpp @@ -1,5 +1,5 @@ // SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -// Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,11 +14,15 @@ // limitations under the License. // // SPDX-License-Identifier: Apache-2.0 +#pragma once -#include "cv/tensor_ops/IImageWarp.h" +#include "gems/hash/sha256.hpp" -namespace cvcore { namespace tensor_ops { +namespace nvidia { +namespace isaac { -IImageWarp::~IImageWarp(){} +// helper to get the hash of a file +gxf::Expected hash_file(const char* path); -}} // namespace cvcore::tensor_ops +} // namespace isaac +} // namespace nvidia diff --git a/isaac_ros_bi3d/gxf/gems/hash/sha256.cpp b/isaac_ros_bi3d/gxf/gems/hash/sha256.cpp new file mode 100644 index 0000000..d804f16 --- /dev/null +++ b/isaac_ros_bi3d/gxf/gems/hash/sha256.cpp @@ -0,0 +1,237 @@ +// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES +// Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 +#include "gems/hash/sha256.hpp" + +#include + +namespace nvidia { +namespace isaac { + +namespace { + +constexpr uint32_t K[64] = { 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, + 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, + 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, + 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, + 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, + 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, + 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, + 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, + 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, + 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, + 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, + 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, + 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, + 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2 }; + +#define DBL_INT_ADD(a, b, c) \ + if ((a) > 0xFFFFFFFF - (c)) { \ + ++(b); \ + } \ + (a) += (c); + +#define ROTLEFT(a, b) (((a) << (b)) | ((a) >> (32 - (b)))) +#define ROTRIGHT(a, b) (((a) >> (b)) | ((a) << (32 - (b)))) + +#define CH(x, y, z) (((x) & (y)) ^ (~(x) & (z))) +#define MAJ(x, y, z) (((x) & (y)) ^ ((x) & (z)) ^ ((y) & (z))) + +#define EP0(x) (ROTRIGHT((x), 2) ^ ROTRIGHT((x), 13) ^ ROTRIGHT((x), 22)) +#define EP1(x) (ROTRIGHT((x), 6) ^ ROTRIGHT((x), 11) ^ ROTRIGHT((x), 25)) +#define SIG0(x) (ROTRIGHT((x), 7) ^ ROTRIGHT((x), 18) ^ ((x) >> 3)) +#define SIG1(x) (ROTRIGHT((x), 17) ^ ROTRIGHT((x), 19) ^ ((x) >> 10)) + +} // namespace + +gxf::Expected SHA256::Hash(const Span data) { + SHA256 hasher; + return hasher.hashData(data) + .and_then([&]() { return hasher.finalize(); }) + .substitute(hasher.hash()); +} + +gxf::Expected SHA256::Hash(const Span data) { + auto span = Span(reinterpret_cast(data.data()), data.size()); + return Hash(span).map(ToString); +} + +void SHA256::Initialize(SHA256_CTX& ctx) { + ctx.datalen = 0; + ctx.bitlen[0] = 0; + ctx.bitlen[1] = 0; + ctx.state[0] = 0x6a09e667; + ctx.state[1] = 0xbb67ae85; + ctx.state[2] = 0x3c6ef372; + ctx.state[3] = 0xa54ff53a; + ctx.state[4] = 0x510e527f; + ctx.state[5] = 0x9b05688c; + ctx.state[6] = 0x1f83d9ab; + ctx.state[7] = 0x5be0cd19; +} + +gxf::Expected SHA256::Update(SHA256_CTX& ctx, const Span data) { + for (auto element : data) { + if (!element) { + return gxf::Unexpected{GXF_FAILURE}; + } + ctx.data[ctx.datalen] = element.value(); + ctx.datalen++; + if (ctx.datalen == 64) { + auto result = Transform(ctx, Span(ctx.data)); + if (!result) { + return gxf::ForwardError(result); + } + DBL_INT_ADD(ctx.bitlen[0], ctx.bitlen[1], 512); + ctx.datalen = 0; + } + } + return gxf::Success; +} + +gxf::Expected SHA256::Transform(SHA256_CTX& ctx, const Span data) { + uint32_t m[64]; + uint32_t i = 0; + uint32_t j = 0; + for (; i < 16; i++) { + if (!data[j] || !data[j + 1] || !data[j + 2] || !data[j + 3]) { + return gxf::Unexpected{GXF_EXCEEDING_PREALLOCATED_SIZE}; + } + m[i] = (static_cast(data[j].value()) << 24 & 0xFF000000) + | (static_cast(data[j + 1].value()) << 16 & 0x00FF0000) + | (static_cast(data[j + 2].value()) << 8 & 0x0000FF00) + | (static_cast(data[j + 3].value()) << 0 & 0x000000FF); + j += 4; + } + for (; i < 64; i++) { + m[i] = SIG1(m[i - 2]) + m[i - 7] + SIG0(m[i - 15]) + m[i - 16]; + } + + uint32_t a = ctx.state[0]; + uint32_t b = ctx.state[1]; + uint32_t c = ctx.state[2]; + uint32_t d = ctx.state[3]; + uint32_t e = ctx.state[4]; + uint32_t f = ctx.state[5]; + uint32_t g = ctx.state[6]; + uint32_t h = ctx.state[7]; + uint32_t t1; + uint32_t t2; + for (i = 0; i < 64; ++i) { + t1 = h + EP1(e) + CH(e, f, g) + K[i] + m[i]; + t2 = EP0(a) + MAJ(a, b, c); + h = g; + g = f; + f = e; + e = d + t1; + d = c; + c = b; + b = a; + a = t1 + t2; + } + + ctx.state[0] += a; + ctx.state[1] += b; + ctx.state[2] += c; + ctx.state[3] += d; + ctx.state[4] += e; + ctx.state[5] += f; + ctx.state[6] += g; + ctx.state[7] += h; + + return gxf::Success; +} + +gxf::Expected SHA256::Finalize(SHA256_CTX& ctx) { + uint32_t i = ctx.datalen; + + if (ctx.datalen < 56) { + ctx.data[i] = 0x80; + i++; + + while (i < 56) { + ctx.data[i] = 0x00; + i++; + } + } else { + ctx.data[i] = 0x80; + i++; + + while (i < 64) { + ctx.data[i] = 0x00; + i++; + } + + auto result = Transform(ctx, Span(ctx.data)); + if (!result) { + return gxf::ForwardError(result); + } + std::memset(ctx.data, 0, 56); + } + + DBL_INT_ADD(ctx.bitlen[0], ctx.bitlen[1], ctx.datalen * 8); + ctx.data[63] = static_cast(ctx.bitlen[0] >> 0 & 0x000000FF); + ctx.data[62] = static_cast(ctx.bitlen[0] >> 8 & 0x000000FF); + ctx.data[61] = static_cast(ctx.bitlen[0] >> 16 & 0x000000FF); + ctx.data[60] = static_cast(ctx.bitlen[0] >> 24 & 0x000000FF); + ctx.data[59] = static_cast(ctx.bitlen[1] >> 0 & 0x000000FF); + ctx.data[58] = static_cast(ctx.bitlen[1] >> 8 & 0x000000FF); + ctx.data[57] = static_cast(ctx.bitlen[1] >> 16 & 0x000000FF); + ctx.data[56] = static_cast(ctx.bitlen[1] >> 24 & 0x000000FF); + + auto result = Transform(ctx, Span(ctx.data)); + if (!result) { + return gxf::ForwardError(result); + } + + Result hash; + for (i = 0; i < 4; ++i) { + hash[i] = static_cast(ctx.state[0] >> (24 - i * 8) & 0x000000FF); + hash[i + 4] = static_cast(ctx.state[1] >> (24 - i * 8) & 0x000000FF); + hash[i + 8] = static_cast(ctx.state[2] >> (24 - i * 8) & 0x000000FF); + hash[i + 12] = static_cast(ctx.state[3] >> (24 - i * 8) & 0x000000FF); + hash[i + 16] = static_cast(ctx.state[4] >> (24 - i * 8) & 0x000000FF); + hash[i + 20] = static_cast(ctx.state[5] >> (24 - i * 8) & 0x000000FF); + hash[i + 24] = static_cast(ctx.state[6] >> (24 - i * 8) & 0x000000FF); + hash[i + 28] = static_cast(ctx.state[7] >> (24 - i * 8) & 0x000000FF); + } + + return hash; +} + +gxf::Expected SHA256::ToString(const Result& hash) { + constexpr char INT2HEX[] = { '0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', 'a', 'b', 'c', 'd', 'e', 'f' }; + + String text; + for (auto&& value : hash) { + const uint8_t upper = static_cast(value >> 4 & 0x0F); + const uint8_t lower = static_cast(value >> 0 & 0x0F); + + auto result = text.append(INT2HEX[upper]) + .and_then([&]() { return text.append(INT2HEX[lower]); }); + if (!result) { + return gxf::Unexpected{GXF_EXCEEDING_PREALLOCATED_SIZE}; + } + } + + return text; +} + +} // namespace isaac +} // namespace nvidia diff --git a/isaac_ros_bi3d/gxf/gems/hash/sha256.hpp b/isaac_ros_bi3d/gxf/gems/hash/sha256.hpp new file mode 100644 index 0000000..0a217c3 --- /dev/null +++ b/isaac_ros_bi3d/gxf/gems/hash/sha256.hpp @@ -0,0 +1,80 @@ +// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES +// Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 +#pragma once + +#include + +#include "common/fixed_string.hpp" +#include "common/span.hpp" +#include "gxf/core/expected.hpp" + +namespace nvidia { +namespace isaac { + +/// The SHA256 class Generates the SHA256 sum of a given string +/// Adapted from https://www.programmingalgorithms.com/algorithm/sha256 +class SHA256 { + public: + static constexpr size_t HASH_LENGTH = 32; + using Result = std::array; + using String = FixedString; + + static gxf::Expected Hash(const Span data); + static gxf::Expected Hash(const Span data); + + SHA256() : context_{}, hash_{} { Initialize(context_); } + + /// Reset hasher to be fed again + void reset() { Initialize(context_); } + + /// Hash a given array + gxf::Expected hashData(const Span data) { return Update(context_, data); } + + /// Finalize computation of the hash, i.e. make solution available through `hash()` + gxf::Expected finalize() { + return Finalize(context_) + .map([&](Result result) { + hash_ = result; + return gxf::Success; + }); + } + + /// Return hashed result + const Result& hash() const { return hash_; } + /// Return base64 encoding of the hash + gxf::Expected toString() const { return ToString(hash_); } + + private: + struct SHA256_CTX { + uint8_t data[64]; + uint32_t datalen; + uint32_t bitlen[2]; + uint32_t state[8]; + }; + + static void Initialize(SHA256_CTX& ctx); + static gxf::Expected Update(SHA256_CTX& ctx, const Span data); + static gxf::Expected Transform(SHA256_CTX& ctx, const Span data); + static gxf::Expected Finalize(SHA256_CTX& ctx); + static gxf::Expected ToString(const Result& hash); + + SHA256_CTX context_; + Result hash_; +}; + +} // namespace isaac +} // namespace nvidia diff --git a/isaac_ros_bi3d/gxf/gems/video_buffer/allocator.hpp b/isaac_ros_bi3d/gxf/gems/video_buffer/allocator.hpp new file mode 100644 index 0000000..dc910d5 --- /dev/null +++ b/isaac_ros_bi3d/gxf/gems/video_buffer/allocator.hpp @@ -0,0 +1,276 @@ +// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES +// Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 +#pragma once + +#include +#include +#include "gxf/multimedia/video.hpp" +#include "gxf/std/allocator.hpp" + +namespace nvidia { +namespace isaac { + +template +struct NoPaddingColorPlanes {}; + +template <> +struct NoPaddingColorPlanes { + explicit NoPaddingColorPlanes(uint32_t width) : planes({gxf::ColorPlane("RGB", 3, width * 3)}) {} + std::array planes; +}; + +template <> +struct NoPaddingColorPlanes { + explicit NoPaddingColorPlanes(uint32_t width) + : planes({nvidia::gxf::ColorPlane("BGR", 3, width * 3)}) {} + std::array planes; +}; + +template <> +struct NoPaddingColorPlanes { + explicit NoPaddingColorPlanes(uint32_t width) + : planes({nvidia::gxf::ColorPlane("RGBA", 4, width * 4)}) {} + std::array planes; +}; + +template <> +struct NoPaddingColorPlanes { + explicit NoPaddingColorPlanes(uint32_t width) + : planes({nvidia::gxf::ColorPlane("BGRA", 4, width * 4)}) {} + std::array planes; +}; + +template <> +struct NoPaddingColorPlanes { + explicit NoPaddingColorPlanes(uint32_t width) + : planes({nvidia::gxf::ColorPlane("RGB", 6, width * 6)}) {} + std::array planes; +}; + +template <> +struct NoPaddingColorPlanes { + explicit NoPaddingColorPlanes(uint32_t width) + : planes({nvidia::gxf::ColorPlane("BGR", 6, width * 6)}) {} + std::array planes; +}; + +template <> +struct NoPaddingColorPlanes { + explicit NoPaddingColorPlanes(uint32_t width) + : planes({nvidia::gxf::ColorPlane("RGB", 12, width * 12)}) {} + std::array planes; +}; + +template <> +struct NoPaddingColorPlanes { + explicit NoPaddingColorPlanes(uint32_t width) + : planes({nvidia::gxf::ColorPlane("BGR", 12, width * 12)}) {} + std::array planes; +}; + +template <> +struct NoPaddingColorPlanes { + explicit NoPaddingColorPlanes(uint32_t width) + : planes({nvidia::gxf::ColorPlane("gray", 1, width)}) {} + std::array planes; +}; + +template <> +struct NoPaddingColorPlanes { + explicit NoPaddingColorPlanes(uint32_t width) + : planes({nvidia::gxf::ColorPlane("gray", 2, width * 2)}) {} + std::array planes; +}; + +template <> +struct NoPaddingColorPlanes { + explicit NoPaddingColorPlanes(uint32_t width) + : planes({nvidia::gxf::ColorPlane("gray", 4, width * 4)}) {} + std::array planes; +}; + +template <> +struct NoPaddingColorPlanes { + explicit NoPaddingColorPlanes(uint32_t width) + : planes({nvidia::gxf::ColorPlane("gray", 4, width * 4)}) {} + std::array planes; +}; + +template <> +struct NoPaddingColorPlanes { + explicit NoPaddingColorPlanes(uint32_t width) + : planes({nvidia::gxf::ColorPlane("Y", 1, width), + nvidia::gxf::ColorPlane("UV", 2, width)}) {} + std::array planes; +}; + +template <> +struct NoPaddingColorPlanes { + explicit NoPaddingColorPlanes(uint32_t width) + : planes({nvidia::gxf::ColorPlane("Y", 1, width), + nvidia::gxf::ColorPlane("UV", 2, width)}) {} + std::array planes; +}; + +template <> +struct NoPaddingColorPlanes { + explicit NoPaddingColorPlanes(uint32_t width) + : planes({nvidia::gxf::ColorPlane("Y", 1, width), + nvidia::gxf::ColorPlane("UV", 2, width * 2)}) {} + std::array planes; +}; + +template <> +struct NoPaddingColorPlanes { + explicit NoPaddingColorPlanes(uint32_t width) + : planes({nvidia::gxf::ColorPlane("Y", 1, width), + nvidia::gxf::ColorPlane("UV", 2, width * 2)}) {} + std::array planes; +}; + +template<> +struct NoPaddingColorPlanes { + explicit NoPaddingColorPlanes(size_t width) + : planes({gxf::ColorPlane("R", 1, width), gxf::ColorPlane("G", 1, width), + gxf::ColorPlane("B", 1, width)}) {} + std::array planes; +}; + +template<> +struct NoPaddingColorPlanes { + explicit NoPaddingColorPlanes(size_t width) + : planes({gxf::ColorPlane("R", 2, width * 2), gxf::ColorPlane("G", 2, width * 2), + gxf::ColorPlane("B", 2, width * 2)}) {} + std::array planes; +}; + +template<> +struct NoPaddingColorPlanes { + explicit NoPaddingColorPlanes(size_t width) + : planes( + {gxf::ColorPlane("R", 4, width * 4), gxf::ColorPlane("G", 4, width * 4), + gxf::ColorPlane("B", 4, width * 4)}) {} + std::array planes; +}; + +template<> +struct NoPaddingColorPlanes { + explicit NoPaddingColorPlanes(size_t width) + : planes({gxf::ColorPlane("B", 1, width), gxf::ColorPlane("G", 1, width), + gxf::ColorPlane("R", 1, width)}) {} + std::array planes; +}; + +template<> +struct NoPaddingColorPlanes { + explicit NoPaddingColorPlanes(size_t width) + : planes( + {gxf::ColorPlane("R", 2, width * 2), gxf::ColorPlane("G", 2, width * 2), + gxf::ColorPlane("B", 2, width * 2)}) {} + std::array planes; +}; + +template<> +struct NoPaddingColorPlanes { + explicit NoPaddingColorPlanes(size_t width) + : planes( + {gxf::ColorPlane("B", 4, width * 4), gxf::ColorPlane("G", 4, width * 4), + gxf::ColorPlane("R", 4, width * 4)}) {} + std::array planes; +}; + +template <> +struct NoPaddingColorPlanes { + explicit NoPaddingColorPlanes(uint32_t width) + : planes({nvidia::gxf::ColorPlane("D", 4, width * 4)}) {} + std::array planes; +}; + +template <> +struct NoPaddingColorPlanes { + explicit NoPaddingColorPlanes(uint32_t width) + : planes({nvidia::gxf::ColorPlane("D", 8, width * 8)}) {} + std::array planes; +}; + +// This includes the list of video buffer formats that supported for the allocator +constexpr bool IsSupportedVideoFormat(const gxf::VideoFormat format) { + return format == gxf::VideoFormat::GXF_VIDEO_FORMAT_RGB || + format == gxf::VideoFormat::GXF_VIDEO_FORMAT_BGR || + format == gxf::VideoFormat::GXF_VIDEO_FORMAT_RGBA || + format == gxf::VideoFormat::GXF_VIDEO_FORMAT_BGRA || + format == gxf::VideoFormat::GXF_VIDEO_FORMAT_RGB16 || + format == gxf::VideoFormat::GXF_VIDEO_FORMAT_BGR16 || + format == gxf::VideoFormat::GXF_VIDEO_FORMAT_RGB32 || + format == gxf::VideoFormat::GXF_VIDEO_FORMAT_BGR32 || + format == gxf::VideoFormat::GXF_VIDEO_FORMAT_GRAY || + format == gxf::VideoFormat::GXF_VIDEO_FORMAT_GRAY16 || + format == gxf::VideoFormat::GXF_VIDEO_FORMAT_GRAY32 || + format == gxf::VideoFormat::GXF_VIDEO_FORMAT_GRAY32F || + format == gxf::VideoFormat::GXF_VIDEO_FORMAT_R8_G8_B8 || + format == gxf::VideoFormat::GXF_VIDEO_FORMAT_B8_G8_R8 || + format == gxf::VideoFormat::GXF_VIDEO_FORMAT_R16_G16_B16 || + format == gxf::VideoFormat::GXF_VIDEO_FORMAT_B16_G16_R16 || + format == gxf::VideoFormat::GXF_VIDEO_FORMAT_R32_G32_B32 || + format == gxf::VideoFormat::GXF_VIDEO_FORMAT_B32_G32_R32 || + format == gxf::VideoFormat::GXF_VIDEO_FORMAT_NV12 || + format == gxf::VideoFormat::GXF_VIDEO_FORMAT_NV12_ER || + format == gxf::VideoFormat::GXF_VIDEO_FORMAT_NV24 || + format == gxf::VideoFormat::GXF_VIDEO_FORMAT_NV24_ER || + format == gxf::VideoFormat::GXF_VIDEO_FORMAT_D32F || + format == gxf::VideoFormat::GXF_VIDEO_FORMAT_D64F; +} + +template::type* = nullptr> +gxf::Expected AllocateUnpaddedVideoBuffer( + gxf::Handle frame, uint32_t width, uint32_t height, + gxf::MemoryStorageType storage_type, + gxf::Handle allocator) { + GXF_LOG_ERROR("Received unsupported video format!"); + return gxf::Unexpected{GXF_FAILURE}; +} + +template::type* = nullptr> +gxf::Expected AllocateUnpaddedVideoBuffer( + gxf::Handle frame, uint32_t width, uint32_t height, + gxf::MemoryStorageType storage_type, gxf::Handle allocator) { + if (width % 2 != 0 || height % 2 != 0) { + GXF_LOG_ERROR( + "Error: expected even width and height but received %u width and %u height", + width, height); + return gxf::Unexpected{GXF_FAILURE}; + } + NoPaddingColorPlanes nopadding_planes(width); + gxf::VideoFormatSize video_format_size; + auto size = video_format_size.size(width, height, nopadding_planes.planes); + std::vector color_planes{ + nopadding_planes.planes.begin(), nopadding_planes.planes.end()}; + gxf::VideoBufferInfo buffer_info{width, height, T, color_planes, + gxf::SurfaceLayout::GXF_SURFACE_LAYOUT_PITCH_LINEAR}; + return frame->resizeCustom(buffer_info, size, storage_type, allocator); +} + +template +gxf::Expected AllocateVideoBuffer( + gxf::Handle frame, uint32_t width, uint32_t height, + gxf::SurfaceLayout layout, gxf::MemoryStorageType storage_type, + gxf::Handle allocator) { + return frame->resize(width, height, layout, storage_type, allocator); +} + +} // namespace isaac +} // namespace nvidia diff --git a/isaac_ros_bi3d/include/isaac_ros_bi3d/bi3d_node.hpp b/isaac_ros_bi3d/include/isaac_ros_bi3d/bi3d_node.hpp index 64f46ca..dddadc2 100644 --- a/isaac_ros_bi3d/include/isaac_ros_bi3d/bi3d_node.hpp +++ b/isaac_ros_bi3d/include/isaac_ros_bi3d/bi3d_node.hpp @@ -22,7 +22,7 @@ #include #include "rclcpp/rclcpp.hpp" -#include "rclcpp/rclcpp/parameter_event_handler.hpp" +#include "rclcpp/parameter_event_handler.hpp" #include "isaac_ros_nitros/nitros_node.hpp" namespace nvidia @@ -52,6 +52,10 @@ class Bi3DNode : public nitros::NitrosNode const gxf_context_t, nitros::NitrosTypeBase &, std::string name, bool pub_disp_values); private: + // Image parameters + const uint16_t image_height_; + const uint16_t image_width_; + // Bi3D model input paramters const std::string featnet_engine_file_path_; const std::vector featnet_input_layers_name_; diff --git a/isaac_ros_bi3d/isaac_ros_bi3d/__init__.py b/isaac_ros_bi3d/isaac_ros_bi3d/__init__.py index e69de29..6bbcb13 100644 --- a/isaac_ros_bi3d/isaac_ros_bi3d/__init__.py +++ b/isaac_ros_bi3d/isaac_ros_bi3d/__init__.py @@ -0,0 +1,16 @@ +# SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES +# Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 diff --git a/isaac_ros_bi3d/launch/isaac_ros_bi3d.launch.py b/isaac_ros_bi3d/launch/isaac_ros_bi3d.launch.py index 1bdebca..131b7a0 100644 --- a/isaac_ros_bi3d/launch/isaac_ros_bi3d.launch.py +++ b/isaac_ros_bi3d/launch/isaac_ros_bi3d.launch.py @@ -24,6 +24,14 @@ def generate_launch_description(): launch_args = [ + DeclareLaunchArgument( + 'image_height', + default_value='576', + description='The height of the input image'), + DeclareLaunchArgument( + 'image_width', + default_value='960', + description='The width of the input image'), DeclareLaunchArgument( 'featnet_engine_file_path', default_value='', @@ -39,6 +47,8 @@ def generate_launch_description(): ] # Bi3DNode parameters + image_height = LaunchConfiguration('image_height') + image_width = LaunchConfiguration('image_width') featnet_engine_file_path = LaunchConfiguration('featnet_engine_file_path') segnet_engine_file_path = LaunchConfiguration('segnet_engine_file_path') max_disparity_values = LaunchConfiguration('max_disparity_values') @@ -48,11 +58,15 @@ def generate_launch_description(): package='isaac_ros_bi3d', plugin='nvidia::isaac_ros::bi3d::Bi3DNode', parameters=[{ + 'image_height': image_height, + 'image_width': image_width, 'featnet_engine_file_path': featnet_engine_file_path, 'segnet_engine_file_path': segnet_engine_file_path, 'max_disparity_values': max_disparity_values}], remappings=[('left_image_bi3d', 'rgb_left'), - ('right_image_bi3d', 'rgb_right')] + ('right_image_bi3d', 'rgb_right'), + ('left_camera_info_bi3d', 'camera_info_left'), + ('right_camera_info_bi3d', 'camera_info_right')] ) container = ComposableNodeContainer( diff --git a/isaac_ros_bi3d/launch/isaac_ros_bi3d_isaac_sim.launch.py b/isaac_ros_bi3d/launch/isaac_ros_bi3d_isaac_sim.launch.py index e2f1f12..ac6dff2 100644 --- a/isaac_ros_bi3d/launch/isaac_ros_bi3d_isaac_sim.launch.py +++ b/isaac_ros_bi3d/launch/isaac_ros_bi3d_isaac_sim.launch.py @@ -46,6 +46,38 @@ def generate_launch_description(): segnet_engine_file_path = LaunchConfiguration('segnet_engine_file_path') max_disparity_values = LaunchConfiguration('max_disparity_values') + image_resize_node_right = ComposableNode( + package='isaac_ros_image_proc', + plugin='nvidia::isaac_ros::image_proc::ResizeNode', + name='image_resize_node_right', + parameters=[{ + 'output_width': 960, + 'output_height': 576, + 'encoding_desired': 'rgb8', + }], + remappings=[ + ('camera_info', 'front_stereo_camera/right_rgb/camerainfo'), + ('image', 'front_stereo_camera/right_rgb/image_raw'), + ('resize/camera_info', 'front_stereo_camera/right_rgb/camerainfo_resize'), + ('resize/image', 'front_stereo_camera/right_rgb/image_resize')] + ) + + image_resize_node_left = ComposableNode( + package='isaac_ros_image_proc', + plugin='nvidia::isaac_ros::image_proc::ResizeNode', + name='image_resize_node_left', + parameters=[{ + 'output_width': 960, + 'output_height': 576, + 'encoding_desired': 'rgb8', + }], + remappings=[ + ('camera_info', 'front_stereo_camera/left_rgb/camerainfo'), + ('image', 'front_stereo_camera/left_rgb/image_raw'), + ('resize/camera_info', 'front_stereo_camera/left_rgb/camerainfo_resize'), + ('resize/image', 'front_stereo_camera/left_rgb/image_resize')] + ) + bi3d_node = ComposableNode( name='bi3d_node', package='isaac_ros_bi3d', @@ -54,20 +86,25 @@ def generate_launch_description(): 'featnet_engine_file_path': featnet_engine_file_path, 'segnet_engine_file_path': segnet_engine_file_path, 'max_disparity_values': max_disparity_values, - 'disparity_values': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 20, 30, 40, 50, 60]}], - remappings=[('left_image_bi3d', 'rgb_left'), - ('right_image_bi3d', 'rgb_right')]) + 'disparity_values': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 20, 30, 40, 50, 60], + 'image_width': 960, + 'image_height': 576 + }], + remappings=[('left_image_bi3d', 'front_stereo_camera/left_rgb/image_resize'), + ('left_camera_info_bi3d', 'front_stereo_camera/left_rgb/camerainfo_resize'), + ('right_image_bi3d', 'front_stereo_camera/right_rgb/image_resize'), + ('right_camera_info_bi3d', 'front_stereo_camera/right_rgb/camerainfo_resize')]) pointcloud_node = ComposableNode( package='isaac_ros_stereo_image_proc', plugin='nvidia::isaac_ros::stereo_image_proc::PointCloudNode', parameters=[{ 'use_color': True, - 'unit_scaling': 1.0 + 'unit_scaling': 0.3 }], - remappings=[('left/image_rect_color', 'rgb_left'), - ('left/camera_info', 'camera_info_left'), - ('right/camera_info', 'camera_info_right'), + remappings=[('left/image_rect_color', 'front_stereo_camera/left_rgb/image_resize'), + ('left/camera_info', 'front_stereo_camera/left_rgb/camerainfo_resize'), + ('right/camera_info', 'front_stereo_camera/right_rgb/camerainfo_resize'), ('disparity', 'bi3d_node/bi3d_output')]) container = ComposableNodeContainer( @@ -75,7 +112,8 @@ def generate_launch_description(): namespace='bi3d', package='rclcpp_components', executable='component_container_mt', - composable_node_descriptions=[bi3d_node, pointcloud_node], + composable_node_descriptions=[bi3d_node, pointcloud_node, + image_resize_node_left, image_resize_node_right], output='screen' ) diff --git a/isaac_ros_bi3d/launch/isaac_ros_bi3d_realsense.launch.py b/isaac_ros_bi3d/launch/isaac_ros_bi3d_realsense.launch.py index c5b5a73..d18c474 100644 --- a/isaac_ros_bi3d/launch/isaac_ros_bi3d_realsense.launch.py +++ b/isaac_ros_bi3d/launch/isaac_ros_bi3d_realsense.launch.py @@ -28,6 +28,14 @@ def generate_launch_description(): launch_args = [ + DeclareLaunchArgument( + 'image_height', + default_value='720', + description='The height of the input image'), + DeclareLaunchArgument( + 'image_width', + default_value='1280', + description='The width of the input image'), DeclareLaunchArgument( 'featnet_engine_file_path', default_value='', @@ -43,6 +51,8 @@ def generate_launch_description(): ] # Bi3DNode parameters + image_height = LaunchConfiguration('image_height') + image_width = LaunchConfiguration('image_width') featnet_engine_file_path = LaunchConfiguration('featnet_engine_file_path') segnet_engine_file_path = LaunchConfiguration('segnet_engine_file_path') max_disparity_values = LaunchConfiguration('max_disparity_values') @@ -52,12 +62,17 @@ def generate_launch_description(): package='isaac_ros_bi3d', plugin='nvidia::isaac_ros::bi3d::Bi3DNode', parameters=[{ + 'image_height': image_height, + 'image_width': image_width, 'featnet_engine_file_path': featnet_engine_file_path, 'segnet_engine_file_path': segnet_engine_file_path, 'max_disparity_values': max_disparity_values}], remappings=[ ('left_image_bi3d', 'infra1/image_rect_raw'), - ('right_image_bi3d', 'infra2/image_rect_raw')]) + ('right_image_bi3d', 'infra2/image_rect_raw'), + ('left_camera_info_bi3d', 'infra1/camera_info'), + ('right_camera_info_bi3d', 'infra2/camera_info')] + ) image_format_converter_node_left = ComposableNode( package='isaac_ros_image_proc', @@ -83,18 +98,6 @@ def generate_launch_description(): ('image', 'infra2/image_rect_raw')] ) - pointcloud_node = ComposableNode( - package='isaac_ros_stereo_image_proc', - plugin='nvidia::isaac_ros::stereo_image_proc::PointCloudNode', - parameters=[{ - 'use_color': True, - 'unit_scaling': 1.0 - }], - remappings=[('left/image_rect_color', 'infra1/image_rect_raw'), - ('left/camera_info', 'infra1/camera_info'), - ('right/camera_info', 'infra2/camera_info')] - ) - # RealSense realsense_config_file_path = os.path.join( get_package_share_directory('isaac_ros_bi3d'), @@ -119,7 +122,6 @@ def generate_launch_description(): composable_node_descriptions=[bi3d_node, image_format_converter_node_left, image_format_converter_node_right, - pointcloud_node, realsense_node], output='screen', remappings=[ diff --git a/isaac_ros_bi3d/launch/isaac_ros_bi3d_zed.launch.py b/isaac_ros_bi3d/launch/isaac_ros_bi3d_zed.launch.py new file mode 100644 index 0000000..59643cf --- /dev/null +++ b/isaac_ros_bi3d/launch/isaac_ros_bi3d_zed.launch.py @@ -0,0 +1,190 @@ +# SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES +# Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +import os + +from ament_index_python.packages import get_package_share_directory + +import launch +from launch.actions import DeclareLaunchArgument +from launch.substitutions import Command, LaunchConfiguration +from launch_ros.actions import ComposableNodeContainer, Node +from launch_ros.descriptions import ComposableNode + + +def generate_launch_description(): + # The zed camera model name. zed, zed2, zed2i, zedm, zedx or zedxm + camera_model = 'zed2i' + + launch_args = [ + DeclareLaunchArgument( + 'image_height', + default_value='1080', + description='The height of the input image'), + DeclareLaunchArgument( + 'image_width', + default_value='3840', + description='The width of the input image'), + DeclareLaunchArgument( + 'featnet_engine_file_path', + default_value='', + description='The absolute path to the Bi3D Featnet TensorRT engine plan'), + DeclareLaunchArgument( + 'segnet_engine_file_path', + default_value='', + description='The absolute path to the Bi3D Segnet TensorRT engine plan'), + DeclareLaunchArgument( + 'max_disparity_values', + default_value='64', + description='The maximum number of disparity values given for Bi3D inference'), + ] + + # Bi3DNode parameters + image_height = LaunchConfiguration('image_height') + image_width = LaunchConfiguration('image_width') + featnet_engine_file_path = LaunchConfiguration('featnet_engine_file_path') + segnet_engine_file_path = LaunchConfiguration('segnet_engine_file_path') + max_disparity_values = LaunchConfiguration('max_disparity_values') + + bi3d_node = ComposableNode( + name='bi3d_node', + package='isaac_ros_bi3d', + plugin='nvidia::isaac_ros::bi3d::Bi3DNode', + parameters=[{ + 'image_height': image_height, + 'image_width': image_width, + 'featnet_engine_file_path': featnet_engine_file_path, + 'segnet_engine_file_path': segnet_engine_file_path, + 'max_disparity_values': max_disparity_values}], + remappings=[ + ('left_image_bi3d', 'zed_node/left/image_rect_color_rgb'), + ('right_image_bi3d', 'zed_node/right/image_rect_color_rgb'), + ('left_camera_info_bi3d', 'zed_node/left/camera_info'), + ('right_camera_info_bi3d', 'zed_node/right/camera_info')] + ) + + pointcloud_node = ComposableNode( + package='isaac_ros_stereo_image_proc', + plugin='nvidia::isaac_ros::stereo_image_proc::PointCloudNode', + parameters=[{ + 'use_color': True, + 'unit_scaling': 1.0, + 'output_height': image_height, + 'output_width': image_width + }], + remappings=[('left/image_rect_color', 'zed_node/left/image_rect_color_rgb'), + ('left/camera_info', 'zed_node/left/camera_info'), + ('right/camera_info', 'zed_node/right/camera_info'), + ('disparity', 'bi3d_node/bi3d_output')] + ) + + image_format_converter_node_left = ComposableNode( + package='isaac_ros_image_proc', + plugin='nvidia::isaac_ros::image_proc::ImageFormatConverterNode', + name='image_format_node_left', + parameters=[{ + 'encoding_desired': 'rgb8', + }], + remappings=[ + ('image_raw', 'zed_node/left/image_rect_color'), + ('image', 'zed_node/left/image_rect_color_rgb')] + ) + + image_format_converter_node_right = ComposableNode( + package='isaac_ros_image_proc', + plugin='nvidia::isaac_ros::image_proc::ImageFormatConverterNode', + name='image_format_node_right', + parameters=[{ + 'encoding_desired': 'rgb8', + }], + remappings=[ + ('image_raw', 'zed_node/right/image_rect_color'), + ('image', 'zed_node/right/image_rect_color_rgb')] + ) + + container = ComposableNodeContainer( + name='bi3d_container', + package='rclcpp_components', + namespace='', + executable='component_container_mt', + composable_node_descriptions=[bi3d_node, + image_format_converter_node_left, + image_format_converter_node_right, + pointcloud_node], + output='screen', + remappings=[ + ('left_image_bi3d', 'zed_node/left/image_rect_color'), + ('right_image_bi3d', 'zed_node/right/image_rect_color')] + ) + + rviz_config_path = os.path.join(get_package_share_directory( + 'isaac_ros_bi3d'), 'config', 'isaac_ros_bi3d_zed.rviz') + + rviz = Node( + package='rviz2', + executable='rviz2', + arguments=['-d', rviz_config_path], + output='screen') + + # URDF/xacro file to be loaded by the Robot State Publisher node + xacro_path = os.path.join( + get_package_share_directory('zed_wrapper'), + 'urdf', 'zed_descr.urdf.xacro' + ) + + # ZED Configurations to be loaded by ZED Node + config_common = os.path.join( + get_package_share_directory('isaac_ros_bi3d'), + 'config', + 'zed.yaml' + ) + + config_camera = os.path.join( + get_package_share_directory('zed_wrapper'), + 'config', + camera_model + '.yaml' + ) + + # Robot State Publisher node + rsp_node = Node( + package='robot_state_publisher', + executable='robot_state_publisher', + name='zed_state_publisher', + output='screen', + parameters=[{ + 'robot_description': Command( + [ + 'xacro', ' ', xacro_path, ' ', + 'camera_name:=', camera_model, ' ', + 'camera_model:=', camera_model + ]) + }] + ) + + # ZED node using manual composition + zed_node = Node( + package='zed_wrapper', + executable='zed_wrapper', + output='screen', + parameters=[ + config_common, # Common parameters + config_camera, # Camera related parameters + ] + ) + + # Add nodes and containers to LaunchDescription + return launch.LaunchDescription(launch_args + [container, rviz, rsp_node, zed_node]) diff --git a/isaac_ros_bi3d/package.xml b/isaac_ros_bi3d/package.xml index bee44a7..3e5f404 100644 --- a/isaac_ros_bi3d/package.xml +++ b/isaac_ros_bi3d/package.xml @@ -21,7 +21,7 @@ SPDX-License-Identifier: Apache-2.0 isaac_ros_bi3d - 0.31.0 + 2.0.0 Bi3D inference network for ROS Hemal Shah @@ -34,7 +34,10 @@ SPDX-License-Identifier: Apache-2.0 rclcpp rclcpp_components isaac_ros_nitros + isaac_ros_image_proc + isaac_ros_stereo_image_proc isaac_ros_nitros_image_type + isaac_ros_nitros_camera_info_type isaac_ros_nitros_disparity_image_type isaac_ros_nitros_bi3d_inference_param_array_type diff --git a/isaac_ros_bi3d/scripts/isaac_ros_bi3d_visualizer.py b/isaac_ros_bi3d/scripts/isaac_ros_bi3d_visualizer.py index 9c20b8b..3fd4df4 100755 --- a/isaac_ros_bi3d/scripts/isaac_ros_bi3d_visualizer.py +++ b/isaac_ros_bi3d/scripts/isaac_ros_bi3d_visualizer.py @@ -45,7 +45,7 @@ def get_args(): help='Topic name for disparity output.') parser.add_argument('--rosbag_path', default='/workspaces/isaac_ros-dev/src/' - 'isaac_ros_proximity_segmentation/resources/' + 'isaac_ros_depth_segmentation/resources/' 'rosbags/bi3dnode_rosbag', help='Absolute path to your rosbag.') args = parser.parse_args() diff --git a/isaac_ros_bi3d/src/bi3d_node.cpp b/isaac_ros_bi3d/src/bi3d_node.cpp index 6340f05..9349d5c 100644 --- a/isaac_ros_bi3d/src/bi3d_node.cpp +++ b/isaac_ros_bi3d/src/bi3d_node.cpp @@ -28,6 +28,7 @@ #include "isaac_ros_nitros_bi3d_inference_param_array_type/nitros_bi3d_inference_param_array.hpp" #include "isaac_ros_nitros_disparity_image_type/nitros_disparity_image.hpp" #include "isaac_ros_nitros_image_type/nitros_image.hpp" +#include "isaac_ros_nitros_camera_info_type/nitros_camera_info.hpp" #include "rclcpp/rclcpp.hpp" #include "rclcpp_components/register_node_macro.hpp" @@ -43,11 +44,16 @@ namespace bi3d using nvidia::gxf::optimizer::GraphIOGroupSupportedDataTypesInfoList; constexpr char INPUT_IMAGE_DEFAULT_TENSOR_FORMAT[] = "nitros_image_rgb8"; +constexpr char INPUT_CAM_INFO_DEFAULT_TENSOR_FORMAT[] = "nitros_camera_info"; -constexpr char INPUT_LEFT_IMAGE_COMPONENT_KEY[] = "sync/data_receiver_left"; +constexpr char INPUT_LEFT_IMAGE_COMPONENT_KEY[] = "sync/left_image_receiver"; constexpr char INPUT_LEFT_IMAGE_TOPIC_NAME[] = "left_image_bi3d"; -constexpr char INPUT_RIGHT_IMAGE_COMPONENT_KEY[] = "sync/data_receiver_right"; +constexpr char INPUT_RIGHT_IMAGE_COMPONENT_KEY[] = "sync/right_image_receiver"; constexpr char INPUT_RIGHT_IMAGE_TOPIC_NAME[] = "right_image_bi3d"; +constexpr char INPUT_LEFT_CAM_INFO_COMPONENT_KEY[] = "sync/left_cam_receiver"; +constexpr char INPUT_LEFT_CAM_INFO_TOPIC_NAME[] = "left_camera_info_bi3d"; +constexpr char INPUT_RIGHT_CAM_INFO_COMPONENT_KEY[] = "sync/right_cam_receiver"; +constexpr char INPUT_RIGHT_CAM_INFO_TOPIC_NAME[] = "right_camera_info_bi3d"; constexpr char INPUT_DISPARITY_COMPONENT_KEY[] = "disparity_roundrobin/data_receiver"; @@ -58,19 +64,18 @@ constexpr char OUTPUT_BI3D_TOPIC_NAME[] = "bi3d_node/bi3d_output"; constexpr char APP_YAML_FILENAME[] = "config/bi3d_node.yaml"; constexpr char PACKAGE_NAME[] = "isaac_ros_bi3d"; -const uint64_t BI3D_BLOCK_SIZE = 2211840; - const std::vector> EXTENSIONS = { {"isaac_ros_gxf", "gxf/lib/std/libgxf_std.so"}, {"isaac_ros_gxf", "gxf/lib/cuda/libgxf_cuda.so"}, {"isaac_ros_gxf", "gxf/lib/multimedia/libgxf_multimedia.so"}, {"isaac_ros_gxf", "gxf/lib/libgxf_synchronization.so"}, - {"isaac_ros_bi3d", "gxf/lib/bi3d/libgxf_cvcore_bi3d.so"}, - {"isaac_ros_bi3d", "gxf/lib/bi3d/libgxf_bi3d_postprocessor.so"} + {"isaac_ros_stereo_image_proc", "gxf/lib/sgm_disparity/libgxf_sgm.so"}, + {"isaac_ros_bi3d", "gxf/lib/bi3d/libgxf_cvcore_bi3d.so"} }; const std::vector PRESET_EXTENSION_SPEC_NAMES = { "isaac_ros_bi3d", }; + const std::vector EXTENSION_SPEC_FILENAMES = {}; const std::vector GENERATOR_RULE_FILENAMES = {}; #pragma GCC diagnostic push @@ -92,6 +97,22 @@ const nitros::NitrosPublisherSubscriberConfigMap CONFIG_MAP = { .topic_name = INPUT_RIGHT_IMAGE_TOPIC_NAME, } }, + {INPUT_LEFT_CAM_INFO_COMPONENT_KEY, + { + .type = nitros::NitrosPublisherSubscriberType::NEGOTIATED, + .qos = rclcpp::QoS(10), + .compatible_data_format = INPUT_CAM_INFO_DEFAULT_TENSOR_FORMAT, + .topic_name = INPUT_LEFT_CAM_INFO_TOPIC_NAME, + } + }, + {INPUT_RIGHT_CAM_INFO_COMPONENT_KEY, + { + .type = nitros::NitrosPublisherSubscriberType::NEGOTIATED, + .qos = rclcpp::QoS(10), + .compatible_data_format = INPUT_CAM_INFO_DEFAULT_TENSOR_FORMAT, + .topic_name = INPUT_RIGHT_CAM_INFO_TOPIC_NAME, + } + }, {INPUT_DISPARITY_COMPONENT_KEY, { .type = nitros::NitrosPublisherSubscriberType::NOOP @@ -175,6 +196,10 @@ Bi3DNode::Bi3DNode(const rclcpp::NodeOptions & options) EXTENSIONS, PACKAGE_NAME), + // Image parameters + image_height_(declare_parameter("image_height", 576)), + image_width_(declare_parameter("image_width", 960)), + // Bi3D model input parameters featnet_engine_file_path_(declare_parameter( "featnet_engine_file_path", @@ -198,7 +223,7 @@ Bi3DNode::Bi3DNode(const rclcpp::NodeOptions & options) // Bi3D extra parameters max_disparity_values_(declare_parameter("max_disparity_values", 64)), disparity_values_(declare_parameter>( - "disparity_values", {10, 20, 30, 40, 50, 60})) + "disparity_values", {5, 10, 15, 20, 25, 30})) { RCLCPP_DEBUG(get_logger(), "[Bi3DNode] Initializing Bi3DNode"); @@ -229,6 +254,7 @@ Bi3DNode::Bi3DNode(const rclcpp::NodeOptions & options) registerSupportedType(); registerSupportedType(); registerSupportedType(); + registerSupportedType(); // This callback will get triggered when there is an attempt to change the parameter auto param_change_callback = @@ -281,54 +307,86 @@ Bi3DNode::Bi3DNode(const rclcpp::NodeOptions & options) startNitrosNode(); } -void Bi3DNode::preLoadGraphCallback() {} +void Bi3DNode::preLoadGraphCallback() +{ + RCLCPP_INFO(get_logger(), "In Bi3D Node preLoadGraphCallback()."); + + // Assemble disparity values string + std::string disparity_values_str = "["; + for (unsigned int i = 0; i < disparity_values_.size(); i++) { + disparity_values_str = disparity_values_str + std::to_string(disparity_values_[i]) + ","; + } + disparity_values_str.pop_back(); + disparity_values_str = disparity_values_str + "]"; + + // Set disparity values string as parameter in Bi3D postprocessor + NitrosNode::preLoadGraphSetParameter( + "bi3d_postprocess", + "nvidia::isaac::bi3d::Bi3DPostprocessor", + "disparity_values", + disparity_values_str); +} void Bi3DNode::postLoadGraphCallback() { // Foward Bi3DNode parameters to GXF component // Bi3D model input parameters getNitrosContext().setParameterStr( - "bi3d_dla0", "nvidia::cvcore::Bi3D", "featnet_engine_file_path", featnet_engine_file_path_); + "bi3d_dla0", "nvidia::isaac::Bi3DInference", "featnet_engine_file_path", + featnet_engine_file_path_); getNitrosContext().setParameter1DStrVector( - "bi3d_dla0", "nvidia::cvcore::Bi3D", "featnet_input_layers_name", featnet_input_layers_name_); + "bi3d_dla0", "nvidia::isaac::Bi3DInference", "featnet_input_layers_name", + featnet_input_layers_name_); getNitrosContext().setParameter1DStrVector( - "bi3d_dla0", "nvidia::cvcore::Bi3D", "featnet_output_layers_name", featnet_output_layers_name_); + "bi3d_dla0", "nvidia::isaac::Bi3DInference", "featnet_output_layers_name", + featnet_output_layers_name_); getNitrosContext().setParameterStr( - "bi3d_dla1", "nvidia::cvcore::Bi3D", "featnet_engine_file_path", featnet_engine_file_path_); + "bi3d_dla1", "nvidia::isaac::Bi3DInference", "featnet_engine_file_path", + featnet_engine_file_path_); getNitrosContext().setParameter1DStrVector( - "bi3d_dla1", "nvidia::cvcore::Bi3D", "featnet_input_layers_name", featnet_input_layers_name_); + "bi3d_dla1", "nvidia::isaac::Bi3DInference", "featnet_input_layers_name", + featnet_input_layers_name_); getNitrosContext().setParameter1DStrVector( - "bi3d_dla1", "nvidia::cvcore::Bi3D", "featnet_output_layers_name", featnet_output_layers_name_); + "bi3d_dla1", "nvidia::isaac::Bi3DInference", "featnet_output_layers_name", + featnet_output_layers_name_); getNitrosContext().setParameterStr( - "bi3d_dla0", "nvidia::cvcore::Bi3D", "segnet_engine_file_path", segnet_engine_file_path_); + "bi3d_dla0", "nvidia::isaac::Bi3DInference", "segnet_engine_file_path", + segnet_engine_file_path_); getNitrosContext().setParameter1DStrVector( - "bi3d_dla0", "nvidia::cvcore::Bi3D", "segnet_input_layers_name", segnet_input_layers_name_); + "bi3d_dla0", "nvidia::isaac::Bi3DInference", "segnet_input_layers_name", + segnet_input_layers_name_); getNitrosContext().setParameter1DStrVector( - "bi3d_dla0", "nvidia::cvcore::Bi3D", "segnet_output_layers_name", segnet_output_layers_name_); + "bi3d_dla0", "nvidia::isaac::Bi3DInference", "segnet_output_layers_name", + segnet_output_layers_name_); getNitrosContext().setParameterStr( - "bi3d_dla1", "nvidia::cvcore::Bi3D", "segnet_engine_file_path", segnet_engine_file_path_); + "bi3d_dla1", "nvidia::isaac::Bi3DInference", "segnet_engine_file_path", + segnet_engine_file_path_); getNitrosContext().setParameter1DStrVector( - "bi3d_dla1", "nvidia::cvcore::Bi3D", "segnet_input_layers_name", segnet_input_layers_name_); + "bi3d_dla1", "nvidia::isaac::Bi3DInference", "segnet_input_layers_name", + segnet_input_layers_name_); getNitrosContext().setParameter1DStrVector( - "bi3d_dla1", "nvidia::cvcore::Bi3D", "segnet_output_layers_name", segnet_output_layers_name_); + "bi3d_dla1", "nvidia::isaac::Bi3DInference", "segnet_output_layers_name", + segnet_output_layers_name_); // Set max allowed number of disparity values getNitrosContext().setParameterUInt64( - "bi3d_dla0", "nvidia::cvcore::Bi3D", "max_disparity_values", max_disparity_values_); + "bi3d_dla0", "nvidia::isaac::Bi3DInference", "max_disparity_values", + max_disparity_values_); getNitrosContext().setParameterUInt64( - "bi3d_dla1", "nvidia::cvcore::Bi3D", "max_disparity_values", max_disparity_values_); + "bi3d_dla1", "nvidia::isaac::Bi3DInference", "max_disparity_values", + max_disparity_values_); // Set Bi3D block memory size depending on maximum number of output disparities getNitrosContext().setParameterUInt64( "bi3d_dla0", "nvidia::gxf::BlockMemoryPool", "block_size", - BI3D_BLOCK_SIZE * max_disparity_values_); + 4 * image_width_ * image_height_ * disparity_values_.size()); getNitrosContext().setParameterUInt64( "bi3d_dla1", "nvidia::gxf::BlockMemoryPool", "block_size", - BI3D_BLOCK_SIZE * max_disparity_values_); + 4 * image_width_ * image_height_ * disparity_values_.size()); getNitrosContext().setParameterUInt64( "bi3d_postprocess", "nvidia::gxf::BlockMemoryPool", "block_size", - BI3D_BLOCK_SIZE * max_disparity_values_); + 4 * image_width_ * image_height_ * disparity_values_.size()); RCLCPP_INFO( get_logger(), "[Bi3DNode] Setting featnet_engine_file_path: %s.", diff --git a/isaac_ros_bi3d/test/camera_info.json b/isaac_ros_bi3d/test/camera_info.json new file mode 100644 index 0000000..1443d8b --- /dev/null +++ b/isaac_ros_bi3d/test/camera_info.json @@ -0,0 +1,51 @@ +{ + "header": { + "frame_id": "tf_camera" + }, + "width": 960, + "height": 576, + "distortion_model": "plumb_bob", + "D": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "K": [ + 434.943999, + 0.000000, + 651.073921, + 0.0, + 431.741273, + 441.878037, + 0.0, + 0.0, + 1.0 + ], + "R": [ + 1.0, + 0.0, + 0.0, + 0.0, + 1.0, + 0.0, + 0.0, + 0.0, + 1.0 + ], + "P": [ + 434.943999, + 0.0, + 651.073921, + 160.0, + 0.0, + 431.741273, + 441.878037, + 0.0, + 0.0, + 0.0, + 1.0, + 0.0 + ] +} \ No newline at end of file diff --git a/isaac_ros_bi3d/test/isaac_ros_bi3d_test.py b/isaac_ros_bi3d/test/isaac_ros_bi3d_test.py index 3c2fe05..3ed670b 100644 --- a/isaac_ros_bi3d/test/isaac_ros_bi3d_test.py +++ b/isaac_ros_bi3d/test/isaac_ros_bi3d_test.py @@ -19,7 +19,7 @@ import subprocess import time -from isaac_ros_test import IsaacROSBaseTest +from isaac_ros_test import IsaacROSBaseTest, JSONConversion from launch_ros.actions import ComposableNodeContainer from launch_ros.descriptions import ComposableNode @@ -27,7 +27,7 @@ import pytest import rclpy -from sensor_msgs.msg import Image +from sensor_msgs.msg import CameraInfo, Image from stereo_msgs.msg import DisparityImage @@ -106,6 +106,7 @@ class IsaacROSBi3DTest(IsaacROSBaseTest): GXF_WAIT_SEC = 10 FEATNET_ENGINE_FILE_PATH = '/tmp/dummy_bi3dnet_featnet.engine' SEGNET_ENGINE_FILE_PATH = '/tmp/dummy_bi3dnet_segnet.engine' + CAMERA_INFO_PATH = os.path.dirname(os.path.realpath(__file__)) + '/camera_info.json' def _create_image(self, name): image = Image() @@ -134,7 +135,8 @@ def test_image_bi3d(self): received_messages = {} self.generate_namespace_lookup( - ['left_image_bi3d', 'right_image_bi3d', 'bi3d_node/bi3d_output']) + ['left_image_bi3d', 'right_image_bi3d', 'left_camera_info_bi3d', + 'right_camera_info_bi3d', 'bi3d_node/bi3d_output']) subs = self.create_logging_subscribers( [('bi3d_node/bi3d_output', DisparityImage)], received_messages) @@ -145,10 +147,17 @@ def test_image_bi3d(self): image_right_pub = self.node.create_publisher( Image, self.namespaces['right_image_bi3d'], self.DEFAULT_QOS ) + camera_info_left = self.node.create_publisher( + CameraInfo, self.namespaces['left_camera_info_bi3d'], self.DEFAULT_QOS + ) + camera_info_right = self.node.create_publisher( + CameraInfo, self.namespaces['right_camera_info_bi3d'], self.DEFAULT_QOS + ) try: left_image = self._create_image('left_image') right_image = self._create_image('right_image') + camera_info = JSONConversion.load_camera_info_from_json(self.CAMERA_INFO_PATH) end_time = time.time() + self.TIMEOUT done = False @@ -156,6 +165,8 @@ def test_image_bi3d(self): while time.time() < end_time: image_left_pub.publish(left_image) image_right_pub.publish(right_image) + camera_info_left.publish(camera_info) + camera_info_right.publish(camera_info) rclpy.spin_once(self.node, timeout_sec=0.1) @@ -164,10 +175,12 @@ def test_image_bi3d(self): break self.assertTrue(done, 'Didnt recieve output on bi3d_node/bi3d_output topic') - disparity = received_messages['bi3d_node/bi3d_output'] - self.assertEqual(disparity.image.encoding, '32FC1') - self.assertEqual(disparity.image.height, self.IMAGE_HEIGHT) - self.assertEqual(disparity.image.width, self.IMAGE_WIDTH) + bi3d_output = received_messages['bi3d_node/bi3d_output'] + self.assertEqual(bi3d_output.image.encoding, '32FC1') + self.assertEqual(bi3d_output.image.height, self.IMAGE_HEIGHT) + self.assertEqual(bi3d_output.image.width, self.IMAGE_WIDTH) + self.assertAlmostEqual(bi3d_output.f, 434.9440002) + self.assertAlmostEqual(bi3d_output.t, -0.3678634) finally: [self.node.destroy_subscription(sub) for sub in subs] diff --git a/isaac_ros_nitros_bi3d_inference_param_array_type/CMakeLists.txt b/isaac_ros_nitros_bi3d_inference_param_array_type/CMakeLists.txt index 3c6118f..551af7b 100644 --- a/isaac_ros_nitros_bi3d_inference_param_array_type/CMakeLists.txt +++ b/isaac_ros_nitros_bi3d_inference_param_array_type/CMakeLists.txt @@ -15,7 +15,7 @@ # # SPDX-License-Identifier: Apache-2.0 -cmake_minimum_required(VERSION 3.23.2) +cmake_minimum_required(VERSION 3.22.1) project(isaac_ros_nitros_bi3d_inference_param_array_type LANGUAGES C CXX) if(CMAKE_COMPILER_IS_GNUCXX OR CMAKE_CXX_COMPILER_ID MATCHES "Clang") @@ -53,6 +53,12 @@ if(BUILD_TESTING) find_package(ament_lint_auto REQUIRED) ament_lint_auto_find_test_dependencies() + + # The FindPythonInterp and FindPythonLibs modules are removed + if(POLICY CMP0148) + cmake_policy(SET CMP0148 OLD) + endif() + find_package(launch_testing_ament_cmake REQUIRED) add_launch_test(test/isaac_ros_nitros_bi3d_inference_param_array_type_test_pol.py TIMEOUT "15") endif() diff --git a/isaac_ros_nitros_bi3d_inference_param_array_type/package.xml b/isaac_ros_nitros_bi3d_inference_param_array_type/package.xml index 5c3d64e..8bf1e6c 100644 --- a/isaac_ros_nitros_bi3d_inference_param_array_type/package.xml +++ b/isaac_ros_nitros_bi3d_inference_param_array_type/package.xml @@ -21,7 +21,7 @@ SPDX-License-Identifier: Apache-2.0 isaac_ros_nitros_bi3d_inference_param_array_type - 0.31.0 + 2.0.0 Isaac ROS Nitros Bi3D Inferece Param Array Type Hemal Shah diff --git a/resources/Isaac_sim_enable_stereo.png b/resources/Isaac_sim_enable_stereo.png deleted file mode 100644 index 69c2126..0000000 --- a/resources/Isaac_sim_enable_stereo.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:8c9ba1412d085b83a42833d12765e6ea331000047bc48849359858c5e9167beb -size 109035 diff --git a/resources/Isaac_sim_play.png b/resources/Isaac_sim_play.png deleted file mode 100644 index 7208c9e..0000000 --- a/resources/Isaac_sim_play.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:e5c0d786681c57ab859790262f19157b22a18a1f4f0bd254fa7f65df12fbe516 -size 784360 diff --git a/resources/Isaac_sim_rviz.png b/resources/Isaac_sim_rviz.png deleted file mode 100644 index eac26e8..0000000 --- a/resources/Isaac_sim_rviz.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:a15788f3b866ad31f75b773bc5f63a09ec73c1ce11110b3ce91bfacaa5436f43 -size 233607 diff --git a/resources/Isaac_sim_set_stereo_offset.png b/resources/Isaac_sim_set_stereo_offset.png deleted file mode 100644 index f6ce4c2..0000000 --- a/resources/Isaac_sim_set_stereo_offset.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:bb562c0a01984e1115ae37811b749b14e2bb1a4429a5e3e2043157ef52d9aa36 -size 112275 diff --git a/resources/Isaac_sim_tutorial.gif b/resources/Isaac_sim_tutorial.gif deleted file mode 100644 index c2af3d2..0000000 --- a/resources/Isaac_sim_tutorial.gif +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:c754df88c336634e4b02925c334d2b199072df640e63af9f7deb78d6db2914b5 -size 8100685 diff --git a/resources/Rviz.png b/resources/Rviz.png deleted file mode 100644 index 2ee4088..0000000 --- a/resources/Rviz.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:f1bbadc50c92acc207914b75d64cfa8c1e8fcb8e028fb6c4f8e2e26e86add7f9 -size 398080 diff --git a/resources/Visualizer_isaac_sim.png b/resources/Visualizer_isaac_sim.png deleted file mode 100644 index 795f43e..0000000 --- a/resources/Visualizer_isaac_sim.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:776ba816d2c6224349be99dd887c5154fa0d100fad7c843345c5ebaf0d855611 -size 22402 diff --git a/resources/bi3d_left.png b/resources/bi3d_left.png deleted file mode 100644 index 4c412b0..0000000 --- a/resources/bi3d_left.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:5ef9d5a906f16c39ad2cbcd3c92f1d52a66f2e88079891afb299658dec621269 -size 631305 diff --git a/resources/bi3d_output_1mps.png b/resources/bi3d_output_1mps.png deleted file mode 100644 index 39170ee..0000000 --- a/resources/bi3d_output_1mps.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:24788a19601da6a7b21380bc296826b03be5c43688068607c80e2f18eb973214 -size 6233 diff --git a/resources/bi3d_output_2mps.png b/resources/bi3d_output_2mps.png deleted file mode 100644 index 006d226..0000000 --- a/resources/bi3d_output_2mps.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:f942d99e40f4c9656693237f4ec730eb44fbe0da82de3f707023fbf64aec3872 -size 11493 diff --git a/resources/bi3d_output_3mps.png b/resources/bi3d_output_3mps.png deleted file mode 100644 index 12705c2..0000000 --- a/resources/bi3d_output_3mps.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:2c0bd80675febe4ba8a0f1f686b90f31ad3f368f713d096fa46bed0235bf7da7 -size 18356 diff --git a/resources/bi3d_visualizer_output.gif b/resources/bi3d_visualizer_output.gif deleted file mode 100644 index 9c89f40..0000000 --- a/resources/bi3d_visualizer_output.gif +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:6d206dc5d51e40cb9537dce5d23c5cb1a47b98ae414fccf72af78d43024bb777 -size 661652 diff --git a/resources/bi3d_visualizer_output.png b/resources/bi3d_visualizer_output.png deleted file mode 100644 index 04f9fc1..0000000 --- a/resources/bi3d_visualizer_output.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:1af3ab0e26feb0daa1a1782ce24481157ade1a4d1d3f7f08acbe22f034ed6a45 -size 14669 diff --git a/resources/disparity_equation.png b/resources/disparity_equation.png deleted file mode 100644 index 1874dfb..0000000 --- a/resources/disparity_equation.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:8d1462c885f2094a5cabcd957405774cb1678477f5657719196d501eaa41393d -size 24355 diff --git a/resources/freespace_example_real.png b/resources/freespace_example_real.png deleted file mode 100644 index 4ae4ff0..0000000 --- a/resources/freespace_example_real.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:be131b6a3d1168f3d927262651edd741b4ac21031955bc394aa2c83e268ae525 -size 540450 diff --git a/resources/freespace_example_segmented.png b/resources/freespace_example_segmented.png deleted file mode 100644 index 60ce989..0000000 --- a/resources/freespace_example_segmented.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:b42c8cdc6ca901d7b3a8acc76be6d0eda9122d8587cfa2ac8c454a9b7d7b478c -size 9634 diff --git a/resources/isaac_ros_bi3d_nodegraph.png b/resources/isaac_ros_bi3d_nodegraph.png deleted file mode 100644 index 90bbac6..0000000 --- a/resources/isaac_ros_bi3d_nodegraph.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:3eaf1d8a07eb69dd7eb5c77cda78065aed1f06157f2b692b4ab2ec87ecc4ec65 -size 43232 diff --git a/resources/isaac_ros_bi3d_real_opt.gif b/resources/isaac_ros_bi3d_real_opt.gif deleted file mode 100644 index 849c64a..0000000 --- a/resources/isaac_ros_bi3d_real_opt.gif +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:c256be263a5f5e62bbcba5950da08cd280c6d70636515a9d07bfb67f6c783380 -size 1361190 diff --git a/resources/quickstart_disparity.png b/resources/quickstart_disparity.png deleted file mode 100644 index 676789d..0000000 --- a/resources/quickstart_disparity.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:364080d877e1c9810866ac029b28ca0d303fc9d226d44b1681aaa10d03364c0f -size 6173 diff --git a/resources/quickstart_rgb.png b/resources/quickstart_rgb.png deleted file mode 100644 index a77f735..0000000 --- a/resources/quickstart_rgb.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:b1b4d0b87ce12bdff57779b2b3272567f7e0a32674d327e7fb5e8c921945122a -size 313818 diff --git a/resources/realsense_camera_position.jpg b/resources/realsense_camera_position.jpg deleted file mode 100644 index e11763f..0000000 --- a/resources/realsense_camera_position.jpg +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:0e5a15c20c0b95d4344e2adea95fbfd198dadfd245b6dd9d4ca6fb5b1f54c9c6 -size 417887 diff --git a/resources/realsense_example.gif b/resources/realsense_example.gif deleted file mode 100644 index 75bb2a1..0000000 --- a/resources/realsense_example.gif +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:6cfbc1f6d3b424d0809c307f6945d0fe4956cc18a50180100ea73fe3d6b9b7cc -size 2137051 diff --git a/resources/realsense_example.png b/resources/realsense_example.png deleted file mode 100644 index 40e0479..0000000 --- a/resources/realsense_example.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:d0e47d33f21d504ee51843a75027d38e3c782256ea584972c4ab4c0e52f0ba18 -size 1466948 diff --git a/resources/safety_zones.png b/resources/safety_zones.png deleted file mode 100644 index b606f1c..0000000 --- a/resources/safety_zones.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:d0110921ec95ab5f8c575151ff62c44ad354b94d3e736f430d75792d04d4e39f -size 29594 diff --git a/resources/visualizer_realsense.png b/resources/visualizer_realsense.png deleted file mode 100644 index 7456b23..0000000 --- a/resources/visualizer_realsense.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:4858a3e885b424a1811e8f7d5d52033241418a025096818d37e647dd2480c726 -size 9347 diff --git a/resources/visualizer_realsense_mono_pair.png b/resources/visualizer_realsense_mono_pair.png deleted file mode 100644 index ac6fde4..0000000 --- a/resources/visualizer_realsense_mono_pair.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:4777f8172e3430ad441a1260e9f3add2234ca9ba661a3864049e92e14d79729f -size 169059