From 2c78425737e02028edc2a898c35ed9cab008a5e3 Mon Sep 17 00:00:00 2001 From: Jaiveer Singh Date: Tue, 10 Dec 2024 17:53:17 -0800 Subject: [PATCH] Isaac ROS 3.2 --- README.md | 4 +- .../apriltag_ros_apriltag_graph-agx_orin.json | 143 +++++++----- ...priltag_ros_apriltag_graph-agx_xavier.json | 74 ------ ...priltag_ros_apriltag_graph-nuc_3060ti.json | 92 -------- ...priltag_ros_apriltag_graph-nuc_4060ti.json | 92 -------- .../apriltag_ros_apriltag_graph-orin_nx.json | 118 ++++++---- .../apriltag_ros_apriltag_graph-x86-4090.json | 133 +++++++++++ .../apriltag_ros_apriltag_graph-x86_4090.json | 92 -------- .../apriltag_ros_apriltag_node-agx_orin.json | 118 ++++++---- ...apriltag_ros_apriltag_node-agx_xavier.json | 74 ------ ...apriltag_ros_apriltag_node-nuc_3060ti.json | 92 -------- ...apriltag_ros_apriltag_node-nuc_4060ti.json | 92 -------- .../apriltag_ros_apriltag_node-orin_nx.json | 120 +++++----- .../apriltag_ros_apriltag_node-x86-4090.json | 133 +++++++++++ .../apriltag_ros_apriltag_node-x86_4090.json | 92 -------- results/image_proc_rectify_node-agx_orin.json | 151 ++++++------ .../image_proc_rectify_node-agx_xavier.json | 90 -------- .../image_proc_rectify_node-nuc_3060ti.json | 90 -------- .../image_proc_rectify_node-nuc_4060ti.json | 90 -------- results/image_proc_rectify_node-orin_nx.json | 153 ++++++------ results/image_proc_rectify_node-x86-4090.json | 131 +++++++++++ results/image_proc_rectify_node-x86_4090.json | 90 -------- ..._transport_h264_decoder_node-agx_orin.json | 51 ---- ...ransport_h264_decoder_node-agx_xavier.json | 51 ---- ...ransport_h264_decoder_node-nuc_3060ti.json | 87 ------- ...ransport_h264_decoder_node-nuc_4060ti.json | 87 ------- ...e_transport_h264_decoder_node-orin_nx.json | 51 ---- ..._transport_h264_decoder_node-x86_4090.json | 69 ------ ..._transport_h264_encoder_node-agx_orin.json | 51 ---- ...ransport_h264_encoder_node-agx_xavier.json | 51 ---- ...ransport_h264_encoder_node-nuc_3060ti.json | 69 ------ ...ransport_h264_encoder_node-nuc_4060ti.json | 69 ------ ...e_transport_h264_encoder_node-orin_nx.json | 51 ---- ..._transport_h264_encoder_node-x86_4090.json | 69 ------ results/stereo_image_proc_graph-agx_orin.json | 153 ++++++------ .../stereo_image_proc_graph-agx_xavier.json | 54 ----- .../stereo_image_proc_graph-nuc_3060ti.json | 90 -------- .../stereo_image_proc_graph-nuc_4060ti.json | 90 -------- .../stereo_image_proc_graph-orin_nano.json | 116 ++++++---- ...tereo_image_proc_graph-orin_nano_emul.json | 72 ------ results/stereo_image_proc_graph-orin_nx.json | 143 +++++++----- results/stereo_image_proc_graph-x86-4090.json | 131 +++++++++++ results/stereo_image_proc_graph-x86_4090.json | 90 -------- results/stereo_image_proc_node-agx_orin.json | 151 ++++++------ .../stereo_image_proc_node-agx_xavier.json | 90 -------- .../stereo_image_proc_node-nuc_3060ti.json | 90 -------- .../stereo_image_proc_node-nuc_4060ti.json | 90 -------- results/stereo_image_proc_node-orin_nano.json | 118 ++++++---- ...stereo_image_proc_node-orin_nano_emul.json | 72 ------ results/stereo_image_proc_node-orin_nx.json | 151 ++++++------ results/stereo_image_proc_node-x86-4090.json | 131 +++++++++++ results/stereo_image_proc_node-x86_4090.json | 90 -------- ros2_benchmark/CMakeLists.txt | 6 + ros2_benchmark/package.xml | 3 +- .../ros2_benchmark/ros2_benchmark_test.py | 10 +- .../ros2_benchmark/utils/cpu_profiler.py | 147 ------------ .../ros2_benchmark/utils/nsys_utility.py | 2 +- .../utils/tegrastats_profiler.py | 2 +- .../ros2_benchmark/utils/x86_profiler.py | 217 ++++++++++++++++++ ros2_benchmark_interfaces/CMakeLists.txt | 6 + ros2_benchmark_interfaces/package.xml | 3 +- 61 files changed, 1850 insertions(+), 3488 deletions(-) delete mode 100644 results/apriltag_ros_apriltag_graph-agx_xavier.json delete mode 100644 results/apriltag_ros_apriltag_graph-nuc_3060ti.json delete mode 100644 results/apriltag_ros_apriltag_graph-nuc_4060ti.json create mode 100644 results/apriltag_ros_apriltag_graph-x86-4090.json delete mode 100644 results/apriltag_ros_apriltag_graph-x86_4090.json delete mode 100644 results/apriltag_ros_apriltag_node-agx_xavier.json delete mode 100644 results/apriltag_ros_apriltag_node-nuc_3060ti.json delete mode 100644 results/apriltag_ros_apriltag_node-nuc_4060ti.json create mode 100644 results/apriltag_ros_apriltag_node-x86-4090.json delete mode 100644 results/apriltag_ros_apriltag_node-x86_4090.json delete mode 100644 results/image_proc_rectify_node-agx_xavier.json delete mode 100644 results/image_proc_rectify_node-nuc_3060ti.json delete mode 100644 results/image_proc_rectify_node-nuc_4060ti.json create mode 100644 results/image_proc_rectify_node-x86-4090.json delete mode 100644 results/image_proc_rectify_node-x86_4090.json delete mode 100644 results/image_transport_h264_decoder_node-agx_orin.json delete mode 100644 results/image_transport_h264_decoder_node-agx_xavier.json delete mode 100644 results/image_transport_h264_decoder_node-nuc_3060ti.json delete mode 100644 results/image_transport_h264_decoder_node-nuc_4060ti.json delete mode 100644 results/image_transport_h264_decoder_node-orin_nx.json delete mode 100644 results/image_transport_h264_decoder_node-x86_4090.json delete mode 100644 results/image_transport_h264_encoder_node-agx_orin.json delete mode 100644 results/image_transport_h264_encoder_node-agx_xavier.json delete mode 100644 results/image_transport_h264_encoder_node-nuc_3060ti.json delete mode 100644 results/image_transport_h264_encoder_node-nuc_4060ti.json delete mode 100644 results/image_transport_h264_encoder_node-orin_nx.json delete mode 100644 results/image_transport_h264_encoder_node-x86_4090.json delete mode 100644 results/stereo_image_proc_graph-agx_xavier.json delete mode 100644 results/stereo_image_proc_graph-nuc_3060ti.json delete mode 100644 results/stereo_image_proc_graph-nuc_4060ti.json delete mode 100644 results/stereo_image_proc_graph-orin_nano_emul.json create mode 100644 results/stereo_image_proc_graph-x86-4090.json delete mode 100644 results/stereo_image_proc_graph-x86_4090.json delete mode 100644 results/stereo_image_proc_node-agx_xavier.json delete mode 100644 results/stereo_image_proc_node-nuc_3060ti.json delete mode 100644 results/stereo_image_proc_node-nuc_4060ti.json delete mode 100644 results/stereo_image_proc_node-orin_nano_emul.json create mode 100644 results/stereo_image_proc_node-x86-4090.json delete mode 100644 results/stereo_image_proc_node-x86_4090.json delete mode 100644 ros2_benchmark/ros2_benchmark/utils/cpu_profiler.py create mode 100644 ros2_benchmark/ros2_benchmark/utils/x86_profiler.py diff --git a/README.md b/README.md index 725a82a..d0d3459 100644 --- a/README.md +++ b/README.md @@ -82,7 +82,7 @@ To use and learn to use `ros2_benchmark`, start by running a sample benchmark. F git clone https://github.com/christianrauch/apriltag_ros.git && \ cd $R2B_WS_HOME && \ sudo apt-get update && \ - rosdep install -i -r --from-paths src --rosdistro humble -y + rosdep update && rosdep install -i -r --from-paths src --rosdistro humble -y ``` 4. Clone and build `image_proc` package with patch to fix incompatible QoS settings. @@ -97,7 +97,7 @@ To use and learn to use `ros2_benchmark`, start by running a sample benchmark. F git apply resize_qos_profile.patch && \ cd $R2B_WS_HOME && \ sudo apt-get update && \ - rosdep install -i -r --from-paths src --rosdistro humble -y && \ + rosdep update && rosdep install -i -r --from-paths src --rosdistro humble -y && \ colcon build --packages-up-to image_proc ``` diff --git a/results/apriltag_ros_apriltag_graph-agx_orin.json b/results/apriltag_ros_apriltag_graph-agx_orin.json index 1c7f055..94cff48 100644 --- a/results/apriltag_ros_apriltag_graph-agx_orin.json +++ b/results/apriltag_ros_apriltag_graph-agx_orin.json @@ -1,74 +1,113 @@ { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4973.735758463542, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 56.29346542189134, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 56.29534904224372, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0.0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 280.0, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 21.679280598958332, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 21.50146484375, - "BasicPerformanceMetrics.MAX_JITTER": 4.03857421875, - "BasicPerformanceMetrics.MIN_JITTER": 0.00732421875, - "BasicPerformanceMetrics.MEAN_JITTER": 1.001255831272482, - "BasicPerformanceMetrics.STD_DEV_JITTER": 0.5097108820386291, - "CPUProfilingMetrics.MAX_CPU_UTIL": 26.166666666666668, - "CPUProfilingMetrics.MIN_CPU_UTIL": 22.375, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 23.43166666666666, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 1.6896382330957904, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 23.647222222222222, + "BasicPerformanceMetrics.RECEIVED_DURATION": 4988.621663411458, + "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 65.51182627074442, + "BasicPerformanceMetrics.MEAN_FRAME_RATE": 63.018793862426925, + "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 11.666666666666666, + "BasicPerformanceMetrics.NUM_FRAMES_SENT": 326.0, + "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 21.232991536458332, + "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 33.685628255208336, + "BasicPerformanceMetrics.MAX_JITTER": 18.889404296875, + "BasicPerformanceMetrics.MIN_JITTER": 0.001953125, + "BasicPerformanceMetrics.MEAN_JITTER": 0.9620017027243589, + "BasicPerformanceMetrics.STD_DEV_JITTER": 2.637805735196718, + "ResourceMetrics.BASELINE_OVERALL_CPU_UTILIZATION": 28.694444444444446, + "ResourceMetrics.MAX_OVERALL_CPU_UTILIZATION": 58.333333333333336, + "ResourceMetrics.MIN_OVERALL_CPU_UTILIZATION": 4.166666666666667, + "ResourceMetrics.MEAN_OVERALL_CPU_UTILIZATION": 25.125, + "ResourceMetrics.STDDEV_OVERALL_CPU_UTILIZATION": 9.531407931584125, + "ResourceMetrics.BASELINE_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MAX_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MIN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_DEVICE_UTILIZATION": 0.0, "10.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4899.74609375, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 10.203899623521941, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 10.204610411094325, + "BasicPerformanceMetrics.RECEIVED_DURATION": 4899.829345703125, + "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 10.203881321287039, + "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 20.900146484375, + "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 20.63330078125, "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, "BasicPerformanceMetrics.NUM_FRAMES_SENT": 50, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 21.81689453125, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 21.4755859375, - "BasicPerformanceMetrics.MAX_JITTER": 2.171142578125, - "BasicPerformanceMetrics.MIN_JITTER": 0.116455078125, - "BasicPerformanceMetrics.MEAN_JITTER": 1.051605224609375, - "BasicPerformanceMetrics.STD_DEV_JITTER": 0.5106522767872985, - "CPUProfilingMetrics.MAX_CPU_UTIL": 5.691666666666666, - "CPUProfilingMetrics.MIN_CPU_UTIL": 4.575, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 4.98, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 0.3881079460379259, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 5.691666666666666 + "BasicPerformanceMetrics.MEAN_FRAME_RATE": 10.20443702673996, + "BasicPerformanceMetrics.MAX_JITTER": 1.828857421875, + "BasicPerformanceMetrics.MIN_JITTER": 0.0078125, + "BasicPerformanceMetrics.MEAN_JITTER": 0.739593505859375, + "BasicPerformanceMetrics.STD_DEV_JITTER": 0.44890855673573404, + "ResourceMetrics.BASELINE_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MAX_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MIN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.BASELINE_OVERALL_CPU_UTILIZATION": 4.166666666666667, + "ResourceMetrics.MEAN_OVERALL_CPU_UTILIZATION": 3.04782196969697, + "ResourceMetrics.STDDEV_OVERALL_CPU_UTILIZATION": 4.434093334961918, + "ResourceMetrics.MAX_OVERALL_CPU_UTILIZATION": 19.416666666666668, + "ResourceMetrics.MIN_OVERALL_CPU_UTILIZATION": 0.0 + }, + "60.0fps": { + "BasicPerformanceMetrics.RECEIVED_DURATION": 4982.797607421875, + "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 60.20186142705357, + "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 21.0888671875, + "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 20.65185546875, + "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, + "BasicPerformanceMetrics.NUM_FRAMES_SENT": 300, + "BasicPerformanceMetrics.MEAN_FRAME_RATE": 60.20714137639268, + "BasicPerformanceMetrics.MAX_JITTER": 2.71630859375, + "BasicPerformanceMetrics.MIN_JITTER": 0.00244140625, + "BasicPerformanceMetrics.MEAN_JITTER": 0.7436695482906879, + "BasicPerformanceMetrics.STD_DEV_JITTER": 0.5609946974035844, + "ResourceMetrics.BASELINE_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MAX_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MIN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.BASELINE_OVERALL_CPU_UTILIZATION": 16.666666666666668, + "ResourceMetrics.MEAN_OVERALL_CPU_UTILIZATION": 22.924242424242422, + "ResourceMetrics.STDDEV_OVERALL_CPU_UTILIZATION": 8.82746764548812, + "ResourceMetrics.MAX_OVERALL_CPU_UTILIZATION": 51.25, + "ResourceMetrics.MIN_OVERALL_CPU_UTILIZATION": 4.166666666666667 }, "30.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4966.509765625, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 30.200823680277093, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 30.202296396999746, + "BasicPerformanceMetrics.RECEIVED_DURATION": 4966.482421875, + "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 30.20080141247733, + "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 21.0478515625, + "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 20.774658203125, "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, "BasicPerformanceMetrics.NUM_FRAMES_SENT": 150, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 21.622802734375, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 21.380615234375, - "BasicPerformanceMetrics.MAX_JITTER": 1.8173828125, - "BasicPerformanceMetrics.MIN_JITTER": 0.025146484375, - "BasicPerformanceMetrics.MEAN_JITTER": 0.9371766786317568, - "BasicPerformanceMetrics.STD_DEV_JITTER": 0.4186160229648217, - "CPUProfilingMetrics.MAX_CPU_UTIL": 13.775, - "CPUProfilingMetrics.MIN_CPU_UTIL": 12.783333333333333, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 13.271666666666667, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 0.4023748942908294, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 13.408333333333333 + "BasicPerformanceMetrics.MEAN_FRAME_RATE": 30.20246268049216, + "BasicPerformanceMetrics.MAX_JITTER": 2.905029296875, + "BasicPerformanceMetrics.MIN_JITTER": 0.000244140625, + "BasicPerformanceMetrics.MEAN_JITTER": 0.805454563450169, + "BasicPerformanceMetrics.STD_DEV_JITTER": 0.597882143729316, + "ResourceMetrics.BASELINE_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MAX_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MIN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.BASELINE_OVERALL_CPU_UTILIZATION": 8.333333333333334, + "ResourceMetrics.MEAN_OVERALL_CPU_UTILIZATION": 13.313920454545451, + "ResourceMetrics.STDDEV_OVERALL_CPU_UTILIZATION": 8.490249552925123, + "ResourceMetrics.MAX_OVERALL_CPU_UTILIZATION": 47.083333333333336, + "ResourceMetrics.MIN_OVERALL_CPU_UTILIZATION": 0.0 }, "custom": { "data_resolution": "HD (1280,720)" }, "metadata": { "BenchmarkMetadata.NAME": "apriltag_ros AprilTag Graph Benchmark", - "BenchmarkMetadata.TEST_FILE_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/scripts/apriltag_ros_apriltag_graph.py", - "BenchmarkMetadata.TEST_DATETIME": "2023-05-25T06:56:47Z", - "BenchmarkMetadata.DEVICE_HOSTNAME": "ros-5bj7cyeix5eolmrzzp4eu34h3y", + "BenchmarkMetadata.TEST_FILE_PATH": "/opt/ros/humble/share/ros2_benchmark/scripts/apriltag_ros_apriltag_graph.py", + "BenchmarkMetadata.TEST_DATETIME": "2024-12-06T12:46:11Z", + "BenchmarkMetadata.DEVICE_HOSTNAME": "a2d202787bde4f26-ade393e2faf849ff", "BenchmarkMetadata.DEVICE_ARCH": "aarch64", - "BenchmarkMetadata.DEVICE_OS": "Linux 5.10.104-tegra #1 SMP PREEMPT Sun Mar 19 07:55:28 PDT 2023", - "BenchmarkMetadata.IDLE_CPU_UTIL": 0.08333333333333333, + "BenchmarkMetadata.DEVICE_OS": "Linux 5.15.148-tegra #1 SMP PREEMPT Thu Sep 12 21:01:54 PDT 2024", + "BenchmarkMetadata.IDLE_CPU_UTIL": 0.15740740740740738, + "BenchmarkMetadata.IDLE_GPU_UTIL": 0.0, "BenchmarkMetadata.BENCHMARK_MODE": 1, - "BenchmarkMetadata.PEAK_THROUGHPUT_PREDICTION": 56.09375, - "BenchmarkMetadata.INPUT_DATA_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets/datasets/r2b_dataset/r2b_storage", + "BenchmarkMetadata.PEAK_THROUGHPUT_PREDICTION": 65.3125, + "BenchmarkMetadata.INPUT_DATA_PATH": "/workspaces/isaac_ros-dev/src/ros2_benchmark/assets/datasets/r2b_dataset/r2b_storage", "BenchmarkMetadata.INPUT_DATA_SIZE": 3087908864, "BenchmarkMetadata.INPUT_DATA_HASH": "5e8f11201fe10dbac7307a8628553e94", "BenchmarkMetadata.INPUT_DATA_START_TIME": 3.0, "BenchmarkMetadata.INPUT_DATA_END_TIME": 3.5, - "BenchmarkMetadata.CONFIG": "ros2_benchmark_config:\n additional_fixed_publisher_rate_tests:\n - 10.0\n - 30.0\n - 60.0\n assets_root: /workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets\n benchmark_duration: 5.0\n benchmark_mode: BenchmarkMode.LOOPING\n benchmark_name: apriltag_ros AprilTag Graph Benchmark\n benchmark_namespace: r2b\n binary_search_acceptable_frame_loss_fraction: 0.05\n binary_search_acceptable_frame_rate_drop: 10.0\n binary_search_duration_fraction: 1.0\n binary_search_terminal_interval_width: 10.0\n cpu_profiling_interval_sec: 1.0\n custom_report_info:\n data_resolution: Resolution(1280, 720, HD)\n default_config_version: 0.30.0\n default_service_future_timeout_sec: 15.0\n enable_cpu_profiler: true\n enable_trial_buffer_preparation: false\n enforce_publisher_rate: false\n input_data_end_time: 3.5\n input_data_path: datasets/r2b_dataset/r2b_storage\n input_data_start_time: 3.0\n linear_scan_acceptable_frame_loss_fraction: 0.05\n linear_scan_acceptable_frame_rate_drop: 5.0\n linear_scan_duration_fraction: 1.0\n linear_scan_step_size: 5.0\n load_data_in_real_time: false\n log_file_name: kpi.json\n log_folder: /osmo/data/output\n monitor_info_list:\n - calculators:\n - class_name: BasicPerformanceCalculator\n config:\n report_prefix: ''\n module_name: ros2_benchmark.basic_performance_calculator\n service_name: start_monitoring0\n play_messages_service_future_timeout_sec: 15.0\n playback_message_buffer_size: 10\n publish_tf_messages_in_set_data: false\n publish_tf_static_messages_in_set_data: false\n publisher_lower_frequency: 10.0\n publisher_upper_frequency: 600.0\n record_data_timeline: false\n revise_timestamps_as_message_ids: false\n set_data_service_future_timeout_sec: 60.0\n setup_service_client_timeout_sec: 20.0\n start_monitoring_service_timeout_sec: 10\n start_recording_service_future_timeout_sec: 35.0\n start_recording_service_timeout_sec: 30\n test_iterations: 5\n" + "BenchmarkMetadata.CONFIG": "ros2_benchmark_config:\n additional_fixed_publisher_rate_tests:\n - 10.0\n - 30.0\n - 60.0\n assets_root: /workspaces/isaac_ros-dev/src/ros2_benchmark/assets\n benchmark_duration: 5.0\n benchmark_mode: BenchmarkMode.LOOPING\n benchmark_name: apriltag_ros AprilTag Graph Benchmark\n benchmark_namespace: r2b\n binary_search_acceptable_frame_loss_fraction: 0.05\n binary_search_acceptable_frame_rate_drop: 10.0\n binary_search_duration_fraction: 1.0\n binary_search_terminal_interval_width: 10.0\n collect_start_timestamps_from_monitors: false\n custom_report_info:\n data_resolution: Resolution(1280, 720, HD)\n default_config_version: 0.30.0\n default_service_future_timeout_sec: 15.0\n enable_resource_profiler: true\n enable_trial_buffer_preparation: false\n enforce_publisher_rate: false\n export_monitor_raw_data: false\n input_data_end_time: 3.5\n input_data_path: datasets/r2b_dataset/r2b_storage\n input_data_start_time: 3.0\n linear_scan_acceptable_frame_loss_fraction: 0.05\n linear_scan_acceptable_frame_rate_drop: 5.0\n linear_scan_duration_fraction: 1.0\n linear_scan_step_size: 5.0\n load_data_in_real_time: false\n log_file_name: kpi.json\n log_folder: /osmo/data/output\n monitor_info_list:\n - calculators:\n - class_name: BasicPerformanceCalculator\n config:\n report_prefix: ''\n module_name: ros2_benchmark.basic_performance_calculator\n service_name: monitor_node0\n play_messages_service_future_timeout_sec: 15.0\n playback_message_buffer_size: 10\n post_trial_run_wait_time_sec: 0.0\n pre_stop_monitoring_wait_time_sec: 5.0\n pre_trial_run_wait_time_sec: 0.0\n publish_tf_messages_in_set_data: false\n publish_tf_static_messages_in_set_data: false\n publisher_lower_frequency: 10.0\n publisher_upper_frequency: 600.0\n record_data_timeline: false\n resource_profiling_interval_sec: 0.01\n revise_timestamps_as_message_ids: false\n ros1_ws: /workspaces/isaac_ros-dev/isaac_ros_1-dev\n set_data_service_future_timeout_sec: 60.0\n setup_service_client_timeout_sec: 20.0\n start_recording_service_future_timeout_sec: 35.0\n start_recording_service_timeout_sec: 30\n test_iterations: 5\n" } } \ No newline at end of file diff --git a/results/apriltag_ros_apriltag_graph-agx_xavier.json b/results/apriltag_ros_apriltag_graph-agx_xavier.json deleted file mode 100644 index f6d9bab..0000000 --- a/results/apriltag_ros_apriltag_graph-agx_xavier.json +++ /dev/null @@ -1,74 +0,0 @@ -{ - "BasicPerformanceMetrics.RECEIVED_DURATION": 4982.469156901042, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 37.85641001734155, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 37.302170642169436, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 2.3333333333333335, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 188.0, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 32.547607421875, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 48.551106770833336, - "BasicPerformanceMetrics.MAX_JITTER": 6.840087890625, - "BasicPerformanceMetrics.MIN_JITTER": 0.00341796875, - "BasicPerformanceMetrics.MEAN_JITTER": 1.068137790845788, - "BasicPerformanceMetrics.STD_DEV_JITTER": 0.8836308837055412, - "CPUProfilingMetrics.MAX_CPU_UTIL": 29.8875, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.5, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 14.451969696969698, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 13.180950309858142, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 27.900000000000002, - "10.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4899.93701171875, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 10.203733889011675, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 10.204212805270636, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 50, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 32.474853515625, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 32.244873046875, - "BasicPerformanceMetrics.MAX_JITTER": 5.814453125, - "BasicPerformanceMetrics.MIN_JITTER": 0.01904296875, - "BasicPerformanceMetrics.MEAN_JITTER": 1.989654541015625, - "BasicPerformanceMetrics.STD_DEV_JITTER": 1.3267283020236127, - "CPUProfilingMetrics.MAX_CPU_UTIL": 12.1625, - "CPUProfilingMetrics.MIN_CPU_UTIL": 7.9125, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 9.5675, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 1.4882372122749785, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 9.387500000000001 - }, - "30.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4965.954345703125, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 30.200304106838292, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 30.205674389614558, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 150, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 31.3466796875, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 30.463623046875, - "BasicPerformanceMetrics.MAX_JITTER": 15.36767578125, - "BasicPerformanceMetrics.MIN_JITTER": 0.029296875, - "BasicPerformanceMetrics.MEAN_JITTER": 2.0854013803842903, - "BasicPerformanceMetrics.STD_DEV_JITTER": 1.874367530717849, - "CPUProfilingMetrics.MAX_CPU_UTIL": 26.012500000000003, - "CPUProfilingMetrics.MIN_CPU_UTIL": 22.85, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 24.1475, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 1.1334515869678778, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 22.85 - }, - "custom": { - "data_resolution": "HD (1280,720)" - }, - "metadata": { - "BenchmarkMetadata.NAME": "apriltag_ros AprilTag Graph Benchmark", - "BenchmarkMetadata.TEST_FILE_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/scripts/apriltag_ros_apriltag_graph.py", - "BenchmarkMetadata.TEST_DATETIME": "2023-05-25T07:45:31Z", - "BenchmarkMetadata.DEVICE_HOSTNAME": "ros-vmbbxwokrng7ferzoro546rhfm", - "BenchmarkMetadata.DEVICE_ARCH": "aarch64", - "BenchmarkMetadata.DEVICE_OS": "Linux 5.10.104-tegra #1 SMP PREEMPT Sun Mar 19 07:55:28 PDT 2023", - "BenchmarkMetadata.IDLE_CPU_UTIL": 2.725, - "BenchmarkMetadata.BENCHMARK_MODE": 1, - "BenchmarkMetadata.PEAK_THROUGHPUT_PREDICTION": 37.65625, - "BenchmarkMetadata.INPUT_DATA_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets/datasets/r2b_dataset/r2b_storage", - "BenchmarkMetadata.INPUT_DATA_SIZE": 3087908864, - "BenchmarkMetadata.INPUT_DATA_HASH": "5e8f11201fe10dbac7307a8628553e94", - "BenchmarkMetadata.INPUT_DATA_START_TIME": 3.0, - "BenchmarkMetadata.INPUT_DATA_END_TIME": 3.5, - "BenchmarkMetadata.CONFIG": "ros2_benchmark_config:\n additional_fixed_publisher_rate_tests:\n - 10.0\n - 30.0\n - 60.0\n assets_root: /workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets\n benchmark_duration: 5.0\n benchmark_mode: BenchmarkMode.LOOPING\n benchmark_name: apriltag_ros AprilTag Graph Benchmark\n benchmark_namespace: r2b\n binary_search_acceptable_frame_loss_fraction: 0.05\n binary_search_acceptable_frame_rate_drop: 10.0\n binary_search_duration_fraction: 1.0\n binary_search_terminal_interval_width: 10.0\n cpu_profiling_interval_sec: 1.0\n custom_report_info:\n data_resolution: Resolution(1280, 720, HD)\n default_config_version: 0.30.0\n default_service_future_timeout_sec: 15.0\n enable_cpu_profiler: true\n enable_trial_buffer_preparation: false\n enforce_publisher_rate: false\n input_data_end_time: 3.5\n input_data_path: datasets/r2b_dataset/r2b_storage\n input_data_start_time: 3.0\n linear_scan_acceptable_frame_loss_fraction: 0.05\n linear_scan_acceptable_frame_rate_drop: 5.0\n linear_scan_duration_fraction: 1.0\n linear_scan_step_size: 5.0\n load_data_in_real_time: false\n log_file_name: kpi.json\n log_folder: /osmo/data/output\n monitor_info_list:\n - calculators:\n - class_name: BasicPerformanceCalculator\n config:\n report_prefix: ''\n module_name: ros2_benchmark.basic_performance_calculator\n service_name: start_monitoring0\n play_messages_service_future_timeout_sec: 15.0\n playback_message_buffer_size: 10\n publish_tf_messages_in_set_data: false\n publish_tf_static_messages_in_set_data: false\n publisher_lower_frequency: 10.0\n publisher_upper_frequency: 600.0\n record_data_timeline: false\n revise_timestamps_as_message_ids: false\n set_data_service_future_timeout_sec: 60.0\n setup_service_client_timeout_sec: 20.0\n start_monitoring_service_timeout_sec: 10\n start_recording_service_future_timeout_sec: 35.0\n start_recording_service_timeout_sec: 30\n test_iterations: 5\n" - } -} \ No newline at end of file diff --git a/results/apriltag_ros_apriltag_graph-nuc_3060ti.json b/results/apriltag_ros_apriltag_graph-nuc_3060ti.json deleted file mode 100644 index 3d8a63c..0000000 --- a/results/apriltag_ros_apriltag_graph-nuc_3060ti.json +++ /dev/null @@ -1,92 +0,0 @@ -{ - "BasicPerformanceMetrics.RECEIVED_DURATION": 4985.373453776042, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 93.16807911970324, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 88.09185976876493, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 25.0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 464.0, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 11.984212239583334, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 16.95068359375, - "BasicPerformanceMetrics.MAX_JITTER": 10.022216796875, - "BasicPerformanceMetrics.MIN_JITTER": 0.00341796875, - "BasicPerformanceMetrics.MEAN_JITTER": 1.4622950354287791, - "BasicPerformanceMetrics.STD_DEV_JITTER": 1.4477781719298513, - "CPUProfilingMetrics.MAX_CPU_UTIL": 14.0, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.1875, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 6.266666666666666, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 6.318488031371672, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 13.414583333333333, - "10.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4906.814697265625, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 10.203097945505593, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 10.189909969060588, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 50, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 12.1640625, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 18.50634765625, - "BasicPerformanceMetrics.MAX_JITTER": 27.393798828125, - "BasicPerformanceMetrics.MIN_JITTER": 0.587890625, - "BasicPerformanceMetrics.MEAN_JITTER": 12.698221842447916, - "BasicPerformanceMetrics.STD_DEV_JITTER": 6.731931814799729, - "CPUProfilingMetrics.MAX_CPU_UTIL": 5.175000000000001, - "CPUProfilingMetrics.MIN_CPU_UTIL": 4.3812500000000005, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 4.7175, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 0.2798827879666774, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 4.875 - }, - "60.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4982.646484375, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 60.19844322749877, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 60.208967451486906, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 300, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 12.275146484375, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 11.404052734375, - "BasicPerformanceMetrics.MAX_JITTER": 17.2734375, - "BasicPerformanceMetrics.MIN_JITTER": 0.031005859375, - "BasicPerformanceMetrics.MEAN_JITTER": 5.148335092019715, - "BasicPerformanceMetrics.STD_DEV_JITTER": 3.8805675598993488, - "CPUProfilingMetrics.MAX_CPU_UTIL": 10.5, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.4375, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 7.293749999999999, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 3.2136873616351256, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 8.1 - }, - "30.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4972.551513671875, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 30.20001909128811, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 30.165600011901272, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 150, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 12.101806640625, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 17.76904296875, - "BasicPerformanceMetrics.MAX_JITTER": 25.796630859375, - "BasicPerformanceMetrics.MIN_JITTER": 0.16552734375, - "BasicPerformanceMetrics.MEAN_JITTER": 9.841232712204391, - "BasicPerformanceMetrics.STD_DEV_JITTER": 6.495626059180888, - "CPUProfilingMetrics.MAX_CPU_UTIL": 6.0625, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.4375, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 4.4624999999999995, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 1.8704493388399128, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 4.6625 - }, - "custom": { - "data_resolution": "HD (1280,720)" - }, - "metadata": { - "BenchmarkMetadata.NAME": "apriltag_ros AprilTag Graph Benchmark", - "BenchmarkMetadata.TEST_FILE_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/scripts/apriltag_ros_apriltag_graph.py", - "BenchmarkMetadata.TEST_DATETIME": "2023-05-25T06:23:07Z", - "BenchmarkMetadata.DEVICE_HOSTNAME": "ros-7pjlirh6cjbn7pnymmyl3q2t64", - "BenchmarkMetadata.DEVICE_ARCH": "x86_64", - "BenchmarkMetadata.DEVICE_OS": "Linux 5.15.0-50-generic #56~20.04.1-Ubuntu SMP Tue Sep 27 15:51:29 UTC 2022", - "BenchmarkMetadata.IDLE_CPU_UTIL": 0.125, - "BenchmarkMetadata.BENCHMARK_MODE": 1, - "BenchmarkMetadata.PEAK_THROUGHPUT_PREDICTION": 92.96875, - "BenchmarkMetadata.INPUT_DATA_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets/datasets/r2b_dataset/r2b_storage", - "BenchmarkMetadata.INPUT_DATA_SIZE": 3087908864, - "BenchmarkMetadata.INPUT_DATA_HASH": "5e8f11201fe10dbac7307a8628553e94", - "BenchmarkMetadata.INPUT_DATA_START_TIME": 3.0, - "BenchmarkMetadata.INPUT_DATA_END_TIME": 3.5, - "BenchmarkMetadata.CONFIG": "ros2_benchmark_config:\n additional_fixed_publisher_rate_tests:\n - 10.0\n - 30.0\n - 60.0\n assets_root: /workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets\n benchmark_duration: 5.0\n benchmark_mode: BenchmarkMode.LOOPING\n benchmark_name: apriltag_ros AprilTag Graph Benchmark\n benchmark_namespace: r2b\n binary_search_acceptable_frame_loss_fraction: 0.05\n binary_search_acceptable_frame_rate_drop: 10.0\n binary_search_duration_fraction: 1.0\n binary_search_terminal_interval_width: 10.0\n cpu_profiling_interval_sec: 1.0\n custom_report_info:\n data_resolution: Resolution(1280, 720, HD)\n default_config_version: 0.30.0\n default_service_future_timeout_sec: 15.0\n enable_cpu_profiler: true\n enable_trial_buffer_preparation: false\n enforce_publisher_rate: false\n input_data_end_time: 3.5\n input_data_path: datasets/r2b_dataset/r2b_storage\n input_data_start_time: 3.0\n linear_scan_acceptable_frame_loss_fraction: 0.05\n linear_scan_acceptable_frame_rate_drop: 5.0\n linear_scan_duration_fraction: 1.0\n linear_scan_step_size: 5.0\n load_data_in_real_time: false\n log_file_name: kpi.json\n log_folder: /osmo/data/output\n monitor_info_list:\n - calculators:\n - class_name: BasicPerformanceCalculator\n config:\n report_prefix: ''\n module_name: ros2_benchmark.basic_performance_calculator\n service_name: start_monitoring0\n play_messages_service_future_timeout_sec: 15.0\n playback_message_buffer_size: 10\n publish_tf_messages_in_set_data: false\n publish_tf_static_messages_in_set_data: false\n publisher_lower_frequency: 10.0\n publisher_upper_frequency: 600.0\n record_data_timeline: false\n revise_timestamps_as_message_ids: false\n set_data_service_future_timeout_sec: 60.0\n setup_service_client_timeout_sec: 20.0\n start_monitoring_service_timeout_sec: 10\n start_recording_service_future_timeout_sec: 35.0\n start_recording_service_timeout_sec: 30\n test_iterations: 5\n" - } -} \ No newline at end of file diff --git a/results/apriltag_ros_apriltag_graph-nuc_4060ti.json b/results/apriltag_ros_apriltag_graph-nuc_4060ti.json deleted file mode 100644 index e7733ef..0000000 --- a/results/apriltag_ros_apriltag_graph-nuc_4060ti.json +++ /dev/null @@ -1,92 +0,0 @@ -{ - "BasicPerformanceMetrics.RECEIVED_DURATION": 4991.265462239583, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 93.16725549558316, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 88.10336864639282, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 24.0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 464.0, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 12.150553385416666, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 23.159098307291668, - "BasicPerformanceMetrics.MAX_JITTER": 7.69482421875, - "BasicPerformanceMetrics.MIN_JITTER": 0.0009765625, - "BasicPerformanceMetrics.MEAN_JITTER": 1.2128141201036866, - "BasicPerformanceMetrics.STD_DEV_JITTER": 1.3064425173366752, - "CPUProfilingMetrics.MAX_CPU_UTIL": 14.1375, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.1875, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 6.216477272727272, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 6.3815919223842315, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 13.174999999999999, - "10.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4906.3662109375, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 10.202975951017743, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 10.190841419162245, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 50, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 12.01025390625, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 17.845458984375, - "BasicPerformanceMetrics.MAX_JITTER": 29.917724609375, - "BasicPerformanceMetrics.MIN_JITTER": 0.54736328125, - "BasicPerformanceMetrics.MEAN_JITTER": 12.245829264322916, - "BasicPerformanceMetrics.STD_DEV_JITTER": 6.351569014979023, - "CPUProfilingMetrics.MAX_CPU_UTIL": 2.78125, - "CPUProfilingMetrics.MIN_CPU_UTIL": 1.825, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 2.2375, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 0.36909941072832947, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 2.78125 - }, - "60.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4987.793701171875, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 60.198334111042115, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 60.14683404598619, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 300, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 12.044921875, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 16.31201171875, - "BasicPerformanceMetrics.MAX_JITTER": 16.479736328125, - "BasicPerformanceMetrics.MIN_JITTER": 0.03466796875, - "BasicPerformanceMetrics.MEAN_JITTER": 4.213605842334312, - "BasicPerformanceMetrics.STD_DEV_JITTER": 3.3857023402973523, - "CPUProfilingMetrics.MAX_CPU_UTIL": 10.556249999999999, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.25, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 7.898958333333334, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 3.4659343239373737, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 9.2625 - }, - "30.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4966.684326171875, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 30.20090236009926, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 30.2012348982151, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 150, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 11.805908203125, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 11.751220703125, - "BasicPerformanceMetrics.MAX_JITTER": 26.599365234375, - "BasicPerformanceMetrics.MIN_JITTER": 0.05029296875, - "BasicPerformanceMetrics.MEAN_JITTER": 9.59183151657517, - "BasicPerformanceMetrics.STD_DEV_JITTER": 6.070004860934021, - "CPUProfilingMetrics.MAX_CPU_UTIL": 6.025, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.36875, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 4.592708333333333, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 1.9198063618536867, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 4.9 - }, - "custom": { - "data_resolution": "HD (1280,720)" - }, - "metadata": { - "BenchmarkMetadata.NAME": "apriltag_ros AprilTag Graph Benchmark", - "BenchmarkMetadata.TEST_FILE_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/scripts/apriltag_ros_apriltag_graph.py", - "BenchmarkMetadata.TEST_DATETIME": "2023-05-25T08:00:15Z", - "BenchmarkMetadata.DEVICE_HOSTNAME": "ros-fdmoqiwj6vgnlbwu3yvwuc53kq", - "BenchmarkMetadata.DEVICE_ARCH": "x86_64", - "BenchmarkMetadata.DEVICE_OS": "Linux 5.15.0-72-generic #79~20.04.1-Ubuntu SMP Thu Apr 20 22:12:07 UTC 2023", - "BenchmarkMetadata.IDLE_CPU_UTIL": 0.125, - "BenchmarkMetadata.BENCHMARK_MODE": 1, - "BenchmarkMetadata.PEAK_THROUGHPUT_PREDICTION": 92.96875, - "BenchmarkMetadata.INPUT_DATA_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets/datasets/r2b_dataset/r2b_storage", - "BenchmarkMetadata.INPUT_DATA_SIZE": 3087908864, - "BenchmarkMetadata.INPUT_DATA_HASH": "5e8f11201fe10dbac7307a8628553e94", - "BenchmarkMetadata.INPUT_DATA_START_TIME": 3.0, - "BenchmarkMetadata.INPUT_DATA_END_TIME": 3.5, - "BenchmarkMetadata.CONFIG": "ros2_benchmark_config:\n additional_fixed_publisher_rate_tests:\n - 10.0\n - 30.0\n - 60.0\n assets_root: /workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets\n benchmark_duration: 5.0\n benchmark_mode: BenchmarkMode.LOOPING\n benchmark_name: apriltag_ros AprilTag Graph Benchmark\n benchmark_namespace: r2b\n binary_search_acceptable_frame_loss_fraction: 0.05\n binary_search_acceptable_frame_rate_drop: 10.0\n binary_search_duration_fraction: 1.0\n binary_search_terminal_interval_width: 10.0\n cpu_profiling_interval_sec: 1.0\n custom_report_info:\n data_resolution: Resolution(1280, 720, HD)\n default_config_version: 0.30.0\n default_service_future_timeout_sec: 15.0\n enable_cpu_profiler: true\n enable_trial_buffer_preparation: false\n enforce_publisher_rate: false\n input_data_end_time: 3.5\n input_data_path: datasets/r2b_dataset/r2b_storage\n input_data_start_time: 3.0\n linear_scan_acceptable_frame_loss_fraction: 0.05\n linear_scan_acceptable_frame_rate_drop: 5.0\n linear_scan_duration_fraction: 1.0\n linear_scan_step_size: 5.0\n load_data_in_real_time: false\n log_file_name: kpi.json\n log_folder: /osmo/data/output\n monitor_info_list:\n - calculators:\n - class_name: BasicPerformanceCalculator\n config:\n report_prefix: ''\n module_name: ros2_benchmark.basic_performance_calculator\n service_name: start_monitoring0\n play_messages_service_future_timeout_sec: 15.0\n playback_message_buffer_size: 10\n publish_tf_messages_in_set_data: false\n publish_tf_static_messages_in_set_data: false\n publisher_lower_frequency: 10.0\n publisher_upper_frequency: 600.0\n record_data_timeline: false\n revise_timestamps_as_message_ids: false\n set_data_service_future_timeout_sec: 60.0\n setup_service_client_timeout_sec: 20.0\n start_monitoring_service_timeout_sec: 10\n start_recording_service_future_timeout_sec: 35.0\n start_recording_service_timeout_sec: 30\n test_iterations: 5\n" - } -} \ No newline at end of file diff --git a/results/apriltag_ros_apriltag_graph-orin_nx.json b/results/apriltag_ros_apriltag_graph-orin_nx.json index a796b17..4db26b7 100644 --- a/results/apriltag_ros_apriltag_graph-orin_nx.json +++ b/results/apriltag_ros_apriltag_graph-orin_nx.json @@ -1,74 +1,90 @@ { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4972.560872395833, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 52.074963264695704, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 52.08583920681544, + "BasicPerformanceMetrics.RECEIVED_DURATION": 4972.870442708333, + "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 56.29343871172974, + "BasicPerformanceMetrics.MEAN_FRAME_RATE": 56.305509630879946, "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0.0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 259.0, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 24.868570963541668, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 23.96630859375, - "BasicPerformanceMetrics.MAX_JITTER": 4.509033203125, - "BasicPerformanceMetrics.MIN_JITTER": 0.001953125, - "BasicPerformanceMetrics.MEAN_JITTER": 0.9405047346181906, - "BasicPerformanceMetrics.STD_DEV_JITTER": 0.703401854170545, - "CPUProfilingMetrics.MAX_CPU_UTIL": 34.912499999999994, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.5, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 30.989444444444445, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 8.544870181090722, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 34.487500000000004, + "BasicPerformanceMetrics.NUM_FRAMES_SENT": 280.0, + "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 24.919759114583332, + "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 23.798421223958332, + "BasicPerformanceMetrics.MAX_JITTER": 4.54736328125, + "BasicPerformanceMetrics.MIN_JITTER": 0.004638671875, + "BasicPerformanceMetrics.MEAN_JITTER": 1.0946352293165467, + "BasicPerformanceMetrics.STD_DEV_JITTER": 0.8534104476701799, + "ResourceMetrics.BASELINE_OVERALL_CPU_UTILIZATION": 32.583333333333336, + "ResourceMetrics.MAX_OVERALL_CPU_UTILIZATION": 74.875, + "ResourceMetrics.MIN_OVERALL_CPU_UTILIZATION": 12.5, + "ResourceMetrics.MEAN_OVERALL_CPU_UTILIZATION": 38.23198784722222, + "ResourceMetrics.STDDEV_OVERALL_CPU_UTILIZATION": 11.502232800792592, + "ResourceMetrics.BASELINE_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MAX_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MIN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_DEVICE_UTILIZATION": 0.0, "10.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4899.739501953125, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 10.203926568598362, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 10.20462413972602, + "BasicPerformanceMetrics.RECEIVED_DURATION": 4899.377197265625, + "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 10.203883863260183, + "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 24.010986328125, + "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 23.293212890625, "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, "BasicPerformanceMetrics.NUM_FRAMES_SENT": 50, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 24.190185546875, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 23.855224609375, - "BasicPerformanceMetrics.MAX_JITTER": 4.31787109375, - "BasicPerformanceMetrics.MIN_JITTER": 0.147216796875, - "BasicPerformanceMetrics.MEAN_JITTER": 1.4572601318359375, - "BasicPerformanceMetrics.STD_DEV_JITTER": 1.103990446361829, - "CPUProfilingMetrics.MAX_CPU_UTIL": 7.9375, - "CPUProfilingMetrics.MIN_CPU_UTIL": 6.5, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 7.3075, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 0.4854508214021273, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 7.3625 + "BasicPerformanceMetrics.MEAN_FRAME_RATE": 10.205378762816085, + "BasicPerformanceMetrics.MAX_JITTER": 3.61181640625, + "BasicPerformanceMetrics.MIN_JITTER": 0.017822265625, + "BasicPerformanceMetrics.MEAN_JITTER": 1.086517333984375, + "BasicPerformanceMetrics.STD_DEV_JITTER": 0.757339252139889, + "ResourceMetrics.BASELINE_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MAX_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MIN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.BASELINE_OVERALL_CPU_UTILIZATION": 16.625, + "ResourceMetrics.MEAN_OVERALL_CPU_UTILIZATION": 6.577020202020202, + "ResourceMetrics.STDDEV_OVERALL_CPU_UTILIZATION": 9.749995258201176, + "ResourceMetrics.MAX_OVERALL_CPU_UTILIZATION": 56.25, + "ResourceMetrics.MIN_OVERALL_CPU_UTILIZATION": 0.0 }, "30.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4964.758544921875, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 30.20090087557052, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 30.212949661655777, + "BasicPerformanceMetrics.RECEIVED_DURATION": 4964.376708984375, + "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 30.200868215975138, + "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 25.661376953125, + "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 23.29345703125, "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, "BasicPerformanceMetrics.NUM_FRAMES_SENT": 150, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 25.812255859375, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 23.83154296875, - "BasicPerformanceMetrics.MAX_JITTER": 4.6767578125, - "BasicPerformanceMetrics.MIN_JITTER": 0.02001953125, - "BasicPerformanceMetrics.MEAN_JITTER": 1.3939687368032094, - "BasicPerformanceMetrics.STD_DEV_JITTER": 0.9666197912861723, - "CPUProfilingMetrics.MAX_CPU_UTIL": 19.525, - "CPUProfilingMetrics.MIN_CPU_UTIL": 17.974999999999998, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 18.814999999999998, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 0.6103994593706651, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 18.837500000000002 + "BasicPerformanceMetrics.MEAN_FRAME_RATE": 30.215273496174184, + "BasicPerformanceMetrics.MAX_JITTER": 3.489013671875, + "BasicPerformanceMetrics.MIN_JITTER": 0.0087890625, + "BasicPerformanceMetrics.MEAN_JITTER": 0.9446246172930743, + "BasicPerformanceMetrics.STD_DEV_JITTER": 0.6990121383465097, + "ResourceMetrics.BASELINE_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MAX_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MIN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.BASELINE_OVERALL_CPU_UTILIZATION": 12.5, + "ResourceMetrics.MEAN_OVERALL_CPU_UTILIZATION": 19.89453125, + "ResourceMetrics.STDDEV_OVERALL_CPU_UTILIZATION": 12.471599613349127, + "ResourceMetrics.MAX_OVERALL_CPU_UTILIZATION": 50.0, + "ResourceMetrics.MIN_OVERALL_CPU_UTILIZATION": 0.0 }, "custom": { "data_resolution": "HD (1280,720)" }, "metadata": { "BenchmarkMetadata.NAME": "apriltag_ros AprilTag Graph Benchmark", - "BenchmarkMetadata.TEST_FILE_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/scripts/apriltag_ros_apriltag_graph.py", - "BenchmarkMetadata.TEST_DATETIME": "2023-05-25T07:34:26Z", - "BenchmarkMetadata.DEVICE_HOSTNAME": "ros-nmo5m6ynwjge7n2hvxm2au7bpu", + "BenchmarkMetadata.TEST_FILE_PATH": "/opt/ros/humble/share/ros2_benchmark/scripts/apriltag_ros_apriltag_graph.py", + "BenchmarkMetadata.TEST_DATETIME": "2024-12-06T13:55:43Z", + "BenchmarkMetadata.DEVICE_HOSTNAME": "a68412aa4de24558-8177cccdbbf34187", "BenchmarkMetadata.DEVICE_ARCH": "aarch64", - "BenchmarkMetadata.DEVICE_OS": "Linux 5.10.104-tegra #1 SMP PREEMPT Wed Mar 15 00:17:07 PDT 2023", - "BenchmarkMetadata.IDLE_CPU_UTIL": 0.625, + "BenchmarkMetadata.DEVICE_OS": "Linux 5.15.148-tegra #1 SMP PREEMPT Thu Sep 12 21:01:54 PDT 2024", + "BenchmarkMetadata.IDLE_CPU_UTIL": 0.3611111111111111, + "BenchmarkMetadata.IDLE_GPU_UTIL": 0.0, "BenchmarkMetadata.BENCHMARK_MODE": 1, - "BenchmarkMetadata.PEAK_THROUGHPUT_PREDICTION": 51.875, - "BenchmarkMetadata.INPUT_DATA_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets/datasets/r2b_dataset/r2b_storage", + "BenchmarkMetadata.PEAK_THROUGHPUT_PREDICTION": 56.09375, + "BenchmarkMetadata.INPUT_DATA_PATH": "/workspaces/isaac_ros-dev/src/ros2_benchmark/assets/datasets/r2b_dataset/r2b_storage", "BenchmarkMetadata.INPUT_DATA_SIZE": 3087908864, "BenchmarkMetadata.INPUT_DATA_HASH": "5e8f11201fe10dbac7307a8628553e94", "BenchmarkMetadata.INPUT_DATA_START_TIME": 3.0, "BenchmarkMetadata.INPUT_DATA_END_TIME": 3.5, - "BenchmarkMetadata.CONFIG": "ros2_benchmark_config:\n additional_fixed_publisher_rate_tests:\n - 10.0\n - 30.0\n - 60.0\n assets_root: /workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets\n benchmark_duration: 5.0\n benchmark_mode: BenchmarkMode.LOOPING\n benchmark_name: apriltag_ros AprilTag Graph Benchmark\n benchmark_namespace: r2b\n binary_search_acceptable_frame_loss_fraction: 0.05\n binary_search_acceptable_frame_rate_drop: 10.0\n binary_search_duration_fraction: 1.0\n binary_search_terminal_interval_width: 10.0\n cpu_profiling_interval_sec: 1.0\n custom_report_info:\n data_resolution: Resolution(1280, 720, HD)\n default_config_version: 0.30.0\n default_service_future_timeout_sec: 15.0\n enable_cpu_profiler: true\n enable_trial_buffer_preparation: false\n enforce_publisher_rate: false\n input_data_end_time: 3.5\n input_data_path: datasets/r2b_dataset/r2b_storage\n input_data_start_time: 3.0\n linear_scan_acceptable_frame_loss_fraction: 0.05\n linear_scan_acceptable_frame_rate_drop: 5.0\n linear_scan_duration_fraction: 1.0\n linear_scan_step_size: 5.0\n load_data_in_real_time: false\n log_file_name: kpi.json\n log_folder: /osmo/data/output\n monitor_info_list:\n - calculators:\n - class_name: BasicPerformanceCalculator\n config:\n report_prefix: ''\n module_name: ros2_benchmark.basic_performance_calculator\n service_name: start_monitoring0\n play_messages_service_future_timeout_sec: 15.0\n playback_message_buffer_size: 10\n publish_tf_messages_in_set_data: false\n publish_tf_static_messages_in_set_data: false\n publisher_lower_frequency: 10.0\n publisher_upper_frequency: 600.0\n record_data_timeline: false\n revise_timestamps_as_message_ids: false\n set_data_service_future_timeout_sec: 60.0\n setup_service_client_timeout_sec: 20.0\n start_monitoring_service_timeout_sec: 10\n start_recording_service_future_timeout_sec: 35.0\n start_recording_service_timeout_sec: 30\n test_iterations: 5\n" + "BenchmarkMetadata.CONFIG": "ros2_benchmark_config:\n additional_fixed_publisher_rate_tests:\n - 10.0\n - 30.0\n - 60.0\n assets_root: /workspaces/isaac_ros-dev/src/ros2_benchmark/assets\n benchmark_duration: 5.0\n benchmark_mode: BenchmarkMode.LOOPING\n benchmark_name: apriltag_ros AprilTag Graph Benchmark\n benchmark_namespace: r2b\n binary_search_acceptable_frame_loss_fraction: 0.05\n binary_search_acceptable_frame_rate_drop: 10.0\n binary_search_duration_fraction: 1.0\n binary_search_terminal_interval_width: 10.0\n collect_start_timestamps_from_monitors: false\n custom_report_info:\n data_resolution: Resolution(1280, 720, HD)\n default_config_version: 0.30.0\n default_service_future_timeout_sec: 15.0\n enable_resource_profiler: true\n enable_trial_buffer_preparation: false\n enforce_publisher_rate: false\n export_monitor_raw_data: false\n input_data_end_time: 3.5\n input_data_path: datasets/r2b_dataset/r2b_storage\n input_data_start_time: 3.0\n linear_scan_acceptable_frame_loss_fraction: 0.05\n linear_scan_acceptable_frame_rate_drop: 5.0\n linear_scan_duration_fraction: 1.0\n linear_scan_step_size: 5.0\n load_data_in_real_time: false\n log_file_name: kpi.json\n log_folder: /osmo/data/output\n monitor_info_list:\n - calculators:\n - class_name: BasicPerformanceCalculator\n config:\n report_prefix: ''\n module_name: ros2_benchmark.basic_performance_calculator\n service_name: monitor_node0\n play_messages_service_future_timeout_sec: 15.0\n playback_message_buffer_size: 10\n post_trial_run_wait_time_sec: 0.0\n pre_stop_monitoring_wait_time_sec: 5.0\n pre_trial_run_wait_time_sec: 0.0\n publish_tf_messages_in_set_data: false\n publish_tf_static_messages_in_set_data: false\n publisher_lower_frequency: 10.0\n publisher_upper_frequency: 600.0\n record_data_timeline: false\n resource_profiling_interval_sec: 0.01\n revise_timestamps_as_message_ids: false\n ros1_ws: /workspaces/isaac_ros-dev/isaac_ros_1-dev\n set_data_service_future_timeout_sec: 60.0\n setup_service_client_timeout_sec: 20.0\n start_recording_service_future_timeout_sec: 35.0\n start_recording_service_timeout_sec: 30\n test_iterations: 5\n" } } \ No newline at end of file diff --git a/results/apriltag_ros_apriltag_graph-x86-4090.json b/results/apriltag_ros_apriltag_graph-x86-4090.json new file mode 100644 index 0000000..f3d576b --- /dev/null +++ b/results/apriltag_ros_apriltag_graph-x86-4090.json @@ -0,0 +1,133 @@ +{ + "BasicPerformanceMetrics.RECEIVED_DURATION": 4993.375162760417, + "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 135.04232228186208, + "BasicPerformanceMetrics.MEAN_FRAME_RATE": 131.5784361371203, + "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 17.0, + "BasicPerformanceMetrics.NUM_FRAMES_SENT": 674.0, + "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 9.446044921875, + "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 11.851155598958334, + "BasicPerformanceMetrics.MAX_JITTER": 1.599853515625, + "BasicPerformanceMetrics.MIN_JITTER": 0.000244140625, + "BasicPerformanceMetrics.MEAN_JITTER": 0.31957374165076335, + "BasicPerformanceMetrics.STD_DEV_JITTER": 0.24765834453807736, + "ResourceMetrics.BASELINE_OVERALL_CPU_UTILIZATION": 7.291666666666667, + "ResourceMetrics.MAX_OVERALL_CPU_UTILIZATION": 18.75, + "ResourceMetrics.MIN_OVERALL_CPU_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_OVERALL_CPU_UTILIZATION": 2.7231865170717042, + "ResourceMetrics.STDDEV_OVERALL_CPU_UTILIZATION": 3.7764654927030405, + "ResourceMetrics.BASELINE_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MAX_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MIN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.BASELINE_DEVICE_MEMORY_UTILIZATION": 1.9400000000000002, + "ResourceMetrics.MAX_DEVICE_MEMORY_UTILIZATION": 1.94, + "ResourceMetrics.MIN_DEVICE_MEMORY_UTILIZATION": 1.94, + "ResourceMetrics.MEAN_DEVICE_MEMORY_UTILIZATION": 1.9400000000000002, + "ResourceMetrics.STDDEV_DEVICE_MEMORY_UTILIZATION": 2.220446049250313e-16, + "10.0fps": { + "BasicPerformanceMetrics.RECEIVED_DURATION": 4899.95166015625, + "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 10.20397893396521, + "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 9.40185546875, + "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 9.30419921875, + "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, + "BasicPerformanceMetrics.NUM_FRAMES_SENT": 50, + "BasicPerformanceMetrics.MEAN_FRAME_RATE": 10.204182299709789, + "BasicPerformanceMetrics.MAX_JITTER": 2.797119140625, + "BasicPerformanceMetrics.MIN_JITTER": 0.065673828125, + "BasicPerformanceMetrics.MEAN_JITTER": 0.6701202392578125, + "BasicPerformanceMetrics.STD_DEV_JITTER": 0.4939989021331712, + "ResourceMetrics.BASELINE_OVERALL_CPU_UTILIZATION": 1.5625, + "ResourceMetrics.MEAN_OVERALL_CPU_UTILIZATION": 0.22849462365591397, + "ResourceMetrics.MAX_OVERALL_CPU_UTILIZATION": 10.9375, + "ResourceMetrics.MIN_OVERALL_CPU_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_OVERALL_CPU_UTILIZATION": 1.1319786024579739, + "ResourceMetrics.BASELINE_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MAX_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MIN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.BASELINE_DEVICE_MEMORY_UTILIZATION": 1.94, + "ResourceMetrics.MEAN_DEVICE_MEMORY_UTILIZATION": 1.9400000000000004, + "ResourceMetrics.STDDEV_DEVICE_MEMORY_UTILIZATION": 4.440892098500626e-16, + "ResourceMetrics.MAX_DEVICE_MEMORY_UTILIZATION": 1.94, + "ResourceMetrics.MIN_DEVICE_MEMORY_UTILIZATION": 1.94 + }, + "60.0fps": { + "BasicPerformanceMetrics.RECEIVED_DURATION": 4982.881103515625, + "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 60.19984998048454, + "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 9.440673828125, + "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 8.920654296875, + "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, + "BasicPerformanceMetrics.NUM_FRAMES_SENT": 300, + "BasicPerformanceMetrics.MEAN_FRAME_RATE": 60.20613251003276, + "BasicPerformanceMetrics.MAX_JITTER": 3.13818359375, + "BasicPerformanceMetrics.MIN_JITTER": 0.000244140625, + "BasicPerformanceMetrics.MEAN_JITTER": 0.810558344693792, + "BasicPerformanceMetrics.STD_DEV_JITTER": 0.6444227876174712, + "ResourceMetrics.BASELINE_OVERALL_CPU_UTILIZATION": 10.9375, + "ResourceMetrics.MEAN_OVERALL_CPU_UTILIZATION": 1.3620094086021506, + "ResourceMetrics.MAX_OVERALL_CPU_UTILIZATION": 14.0625, + "ResourceMetrics.MIN_OVERALL_CPU_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_OVERALL_CPU_UTILIZATION": 2.5320604434849514, + "ResourceMetrics.BASELINE_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MAX_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MIN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.BASELINE_DEVICE_MEMORY_UTILIZATION": 1.94, + "ResourceMetrics.MEAN_DEVICE_MEMORY_UTILIZATION": 1.9400000000000004, + "ResourceMetrics.STDDEV_DEVICE_MEMORY_UTILIZATION": 4.440892098500626e-16, + "ResourceMetrics.MAX_DEVICE_MEMORY_UTILIZATION": 1.94, + "ResourceMetrics.MIN_DEVICE_MEMORY_UTILIZATION": 1.94 + }, + "30.0fps": { + "BasicPerformanceMetrics.RECEIVED_DURATION": 4967.405029296875, + "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 30.200893452929, + "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 9.074462890625, + "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 9.739013671875, + "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, + "BasicPerformanceMetrics.NUM_FRAMES_SENT": 150, + "BasicPerformanceMetrics.MEAN_FRAME_RATE": 30.196853108479488, + "BasicPerformanceMetrics.MAX_JITTER": 2.1923828125, + "BasicPerformanceMetrics.MIN_JITTER": 0.001953125, + "BasicPerformanceMetrics.MEAN_JITTER": 0.6501184411951013, + "BasicPerformanceMetrics.STD_DEV_JITTER": 0.4726673765865835, + "ResourceMetrics.BASELINE_OVERALL_CPU_UTILIZATION": 6.25, + "ResourceMetrics.MEAN_OVERALL_CPU_UTILIZATION": 0.7762096774193549, + "ResourceMetrics.MAX_OVERALL_CPU_UTILIZATION": 10.9375, + "ResourceMetrics.MIN_OVERALL_CPU_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_OVERALL_CPU_UTILIZATION": 1.9310685524917226, + "ResourceMetrics.BASELINE_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MAX_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MIN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.BASELINE_DEVICE_MEMORY_UTILIZATION": 1.94, + "ResourceMetrics.MEAN_DEVICE_MEMORY_UTILIZATION": 1.9400000000000004, + "ResourceMetrics.STDDEV_DEVICE_MEMORY_UTILIZATION": 4.440892098500626e-16, + "ResourceMetrics.MAX_DEVICE_MEMORY_UTILIZATION": 1.94, + "ResourceMetrics.MIN_DEVICE_MEMORY_UTILIZATION": 1.94 + }, + "custom": { + "data_resolution": "HD (1280,720)" + }, + "metadata": { + "BenchmarkMetadata.NAME": "apriltag_ros AprilTag Graph Benchmark", + "BenchmarkMetadata.TEST_FILE_PATH": "/opt/ros/humble/share/ros2_benchmark/scripts/apriltag_ros_apriltag_graph.py", + "BenchmarkMetadata.TEST_DATETIME": "2024-12-06T12:38:33Z", + "BenchmarkMetadata.DEVICE_HOSTNAME": "6694ebf0a5ea4c07-736a21e6f05e4c3f", + "BenchmarkMetadata.DEVICE_ARCH": "x86_64", + "BenchmarkMetadata.DEVICE_OS": "Linux 6.8.0-48-generic #48~22.04.1-Ubuntu SMP PREEMPT_DYNAMIC Mon Oct 7 11:24:13 UTC 2", + "BenchmarkMetadata.IDLE_CPU_UTIL": 0.0625, + "BenchmarkMetadata.IDLE_GPU_UTIL": 0, + "BenchmarkMetadata.BENCHMARK_MODE": 1, + "BenchmarkMetadata.PEAK_THROUGHPUT_PREDICTION": 134.84375, + "BenchmarkMetadata.INPUT_DATA_PATH": "/workspaces/isaac_ros-dev/src/ros2_benchmark/assets/datasets/r2b_dataset/r2b_storage", + "BenchmarkMetadata.INPUT_DATA_SIZE": 3087908864, + "BenchmarkMetadata.INPUT_DATA_HASH": "5e8f11201fe10dbac7307a8628553e94", + "BenchmarkMetadata.INPUT_DATA_START_TIME": 3.0, + "BenchmarkMetadata.INPUT_DATA_END_TIME": 3.5, + "BenchmarkMetadata.CONFIG": "ros2_benchmark_config:\n additional_fixed_publisher_rate_tests:\n - 10.0\n - 30.0\n - 60.0\n assets_root: /workspaces/isaac_ros-dev/src/ros2_benchmark/assets\n benchmark_duration: 5.0\n benchmark_mode: BenchmarkMode.LOOPING\n benchmark_name: apriltag_ros AprilTag Graph Benchmark\n benchmark_namespace: r2b\n binary_search_acceptable_frame_loss_fraction: 0.05\n binary_search_acceptable_frame_rate_drop: 10.0\n binary_search_duration_fraction: 1.0\n binary_search_terminal_interval_width: 10.0\n collect_start_timestamps_from_monitors: false\n custom_report_info:\n data_resolution: Resolution(1280, 720, HD)\n default_config_version: 0.30.0\n default_service_future_timeout_sec: 15.0\n enable_resource_profiler: true\n enable_trial_buffer_preparation: false\n enforce_publisher_rate: false\n export_monitor_raw_data: false\n input_data_end_time: 3.5\n input_data_path: datasets/r2b_dataset/r2b_storage\n input_data_start_time: 3.0\n linear_scan_acceptable_frame_loss_fraction: 0.05\n linear_scan_acceptable_frame_rate_drop: 5.0\n linear_scan_duration_fraction: 1.0\n linear_scan_step_size: 5.0\n load_data_in_real_time: false\n log_file_name: kpi.json\n log_folder: /osmo/data/output\n monitor_info_list:\n - calculators:\n - class_name: BasicPerformanceCalculator\n config:\n report_prefix: ''\n module_name: ros2_benchmark.basic_performance_calculator\n service_name: monitor_node0\n play_messages_service_future_timeout_sec: 15.0\n playback_message_buffer_size: 10\n post_trial_run_wait_time_sec: 0.0\n pre_stop_monitoring_wait_time_sec: 5.0\n pre_trial_run_wait_time_sec: 0.0\n publish_tf_messages_in_set_data: false\n publish_tf_static_messages_in_set_data: false\n publisher_lower_frequency: 10.0\n publisher_upper_frequency: 600.0\n record_data_timeline: false\n resource_profiling_interval_sec: 0.01\n revise_timestamps_as_message_ids: false\n ros1_ws: /workspaces/isaac_ros-dev/isaac_ros_1-dev\n set_data_service_future_timeout_sec: 60.0\n setup_service_client_timeout_sec: 20.0\n start_recording_service_future_timeout_sec: 35.0\n start_recording_service_timeout_sec: 30\n test_iterations: 5\n" + } +} \ No newline at end of file diff --git a/results/apriltag_ros_apriltag_graph-x86_4090.json b/results/apriltag_ros_apriltag_graph-x86_4090.json deleted file mode 100644 index 3a3a224..0000000 --- a/results/apriltag_ros_apriltag_graph-x86_4090.json +++ /dev/null @@ -1,92 +0,0 @@ -{ - "BasicPerformanceMetrics.RECEIVED_DURATION": 4990.48583984375, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 102.38726014774674, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 96.98800488478666, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 26.0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 510.0, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 11.273193359375, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 20.893229166666668, - "BasicPerformanceMetrics.MAX_JITTER": 9.983642578125, - "BasicPerformanceMetrics.MIN_JITTER": 0.00048828125, - "BasicPerformanceMetrics.MEAN_JITTER": 0.8198089281054279, - "BasicPerformanceMetrics.STD_DEV_JITTER": 1.0462663986132987, - "CPUProfilingMetrics.MAX_CPU_UTIL": 14.525, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.125, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 6.527272727272727, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 6.62610895305043, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 13.972916666666668, - "10.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4924.33642578125, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 10.202866666766303, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 10.153652325260751, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 50, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 10.722900390625, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 34.475830078125, - "BasicPerformanceMetrics.MAX_JITTER": 32.800048828125, - "BasicPerformanceMetrics.MIN_JITTER": 0.120849609375, - "BasicPerformanceMetrics.MEAN_JITTER": 9.436823527018229, - "BasicPerformanceMetrics.STD_DEV_JITTER": 9.183527788876939, - "CPUProfilingMetrics.MAX_CPU_UTIL": 3.225, - "CPUProfilingMetrics.MIN_CPU_UTIL": 2.4875000000000003, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 2.82375, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 0.2686889465534449, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 3.225 - }, - "60.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4990.663818359375, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 60.19713975406764, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 60.112243765323726, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 300, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 10.76708984375, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 17.805419921875, - "BasicPerformanceMetrics.MAX_JITTER": 22.48828125, - "BasicPerformanceMetrics.MIN_JITTER": 0.014892578125, - "BasicPerformanceMetrics.MEAN_JITTER": 5.455696669200923, - "BasicPerformanceMetrics.STD_DEV_JITTER": 4.378750749357643, - "CPUProfilingMetrics.MAX_CPU_UTIL": 10.418750000000001, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.125, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 8.105208333333335, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 3.6118176559059414, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 9.65625 - }, - "30.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4983.25439453125, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 30.19945204509863, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 30.10081126193634, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 150, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 10.803466796875, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 27.080322265625, - "BasicPerformanceMetrics.MAX_JITTER": 48.0478515625, - "BasicPerformanceMetrics.MIN_JITTER": 0.107666015625, - "BasicPerformanceMetrics.MEAN_JITTER": 16.132240089210303, - "BasicPerformanceMetrics.STD_DEV_JITTER": 11.200535769358511, - "CPUProfilingMetrics.MAX_CPU_UTIL": 8.69375, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.875, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 6.808333333333334, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 2.6838436044329326, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 7.487500000000001 - }, - "custom": { - "data_resolution": "HD (1280,720)" - }, - "metadata": { - "BenchmarkMetadata.NAME": "apriltag_ros AprilTag Graph Benchmark", - "BenchmarkMetadata.TEST_FILE_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/scripts/apriltag_ros_apriltag_graph.py", - "BenchmarkMetadata.TEST_DATETIME": "2023-05-25T06:38:07Z", - "BenchmarkMetadata.DEVICE_HOSTNAME": "ros-q3f7ou7bi5arff2jcxiiue3nsq", - "BenchmarkMetadata.DEVICE_ARCH": "x86_64", - "BenchmarkMetadata.DEVICE_OS": "Linux 5.4.0-131-generic #147-Ubuntu SMP Fri Oct 14 17:07:22 UTC 2022", - "BenchmarkMetadata.IDLE_CPU_UTIL": 0.25, - "BenchmarkMetadata.BENCHMARK_MODE": 1, - "BenchmarkMetadata.PEAK_THROUGHPUT_PREDICTION": 102.1875, - "BenchmarkMetadata.INPUT_DATA_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets/datasets/r2b_dataset/r2b_storage", - "BenchmarkMetadata.INPUT_DATA_SIZE": 3087908864, - "BenchmarkMetadata.INPUT_DATA_HASH": "5e8f11201fe10dbac7307a8628553e94", - "BenchmarkMetadata.INPUT_DATA_START_TIME": 3.0, - "BenchmarkMetadata.INPUT_DATA_END_TIME": 3.5, - "BenchmarkMetadata.CONFIG": "ros2_benchmark_config:\n additional_fixed_publisher_rate_tests:\n - 10.0\n - 30.0\n - 60.0\n assets_root: /workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets\n benchmark_duration: 5.0\n benchmark_mode: BenchmarkMode.LOOPING\n benchmark_name: apriltag_ros AprilTag Graph Benchmark\n benchmark_namespace: r2b\n binary_search_acceptable_frame_loss_fraction: 0.05\n binary_search_acceptable_frame_rate_drop: 10.0\n binary_search_duration_fraction: 1.0\n binary_search_terminal_interval_width: 10.0\n cpu_profiling_interval_sec: 1.0\n custom_report_info:\n data_resolution: Resolution(1280, 720, HD)\n default_config_version: 0.30.0\n default_service_future_timeout_sec: 15.0\n enable_cpu_profiler: true\n enable_trial_buffer_preparation: false\n enforce_publisher_rate: false\n input_data_end_time: 3.5\n input_data_path: datasets/r2b_dataset/r2b_storage\n input_data_start_time: 3.0\n linear_scan_acceptable_frame_loss_fraction: 0.05\n linear_scan_acceptable_frame_rate_drop: 5.0\n linear_scan_duration_fraction: 1.0\n linear_scan_step_size: 5.0\n load_data_in_real_time: false\n log_file_name: kpi.json\n log_folder: /osmo/data/output\n monitor_info_list:\n - calculators:\n - class_name: BasicPerformanceCalculator\n config:\n report_prefix: ''\n module_name: ros2_benchmark.basic_performance_calculator\n service_name: start_monitoring0\n play_messages_service_future_timeout_sec: 15.0\n playback_message_buffer_size: 10\n publish_tf_messages_in_set_data: false\n publish_tf_static_messages_in_set_data: false\n publisher_lower_frequency: 10.0\n publisher_upper_frequency: 600.0\n record_data_timeline: false\n revise_timestamps_as_message_ids: false\n set_data_service_future_timeout_sec: 60.0\n setup_service_client_timeout_sec: 20.0\n start_monitoring_service_timeout_sec: 10\n start_recording_service_future_timeout_sec: 35.0\n start_recording_service_timeout_sec: 30\n test_iterations: 5\n" - } -} \ No newline at end of file diff --git a/results/apriltag_ros_apriltag_node-agx_orin.json b/results/apriltag_ros_apriltag_node-agx_orin.json index c036c1e..7c9e26e 100644 --- a/results/apriltag_ros_apriltag_node-agx_orin.json +++ b/results/apriltag_ros_apriltag_node-agx_orin.json @@ -1,74 +1,90 @@ { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4973.333821614583, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 56.293862392042236, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 56.300263138291086, + "BasicPerformanceMetrics.RECEIVED_DURATION": 4970.477864583333, + "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 42.8569954894877, + "BasicPerformanceMetrics.MEAN_FRAME_RATE": 42.85302256978, "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0.0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 280.0, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 17.834798177083332, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 17.251871744791668, - "BasicPerformanceMetrics.MAX_JITTER": 3.075439453125, - "BasicPerformanceMetrics.MIN_JITTER": 0.00244140625, - "BasicPerformanceMetrics.MEAN_JITTER": 0.9739972670301259, - "BasicPerformanceMetrics.STD_DEV_JITTER": 0.8940905442097294, - "CPUProfilingMetrics.MAX_CPU_UTIL": 12.066666666666665, - "CPUProfilingMetrics.MIN_CPU_UTIL": 9.358333333333333, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 10.386111111111111, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 0.5494207380508261, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 10.216666666666667, + "BasicPerformanceMetrics.NUM_FRAMES_SENT": 213.0, + "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 23.423990885416668, + "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 23.9775390625, + "BasicPerformanceMetrics.MAX_JITTER": 3.9765625, + "BasicPerformanceMetrics.MIN_JITTER": 0.001708984375, + "BasicPerformanceMetrics.MEAN_JITTER": 1.2821640810130333, + "BasicPerformanceMetrics.STD_DEV_JITTER": 1.096296311679634, + "ResourceMetrics.BASELINE_OVERALL_CPU_UTILIZATION": 6.472222222222222, + "ResourceMetrics.MAX_OVERALL_CPU_UTILIZATION": 36.083333333333336, + "ResourceMetrics.MIN_OVERALL_CPU_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_OVERALL_CPU_UTILIZATION": 10.220486111111112, + "ResourceMetrics.STDDEV_OVERALL_CPU_UTILIZATION": 4.6915455273442985, + "ResourceMetrics.BASELINE_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MAX_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MIN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_DEVICE_UTILIZATION": 0.0, "10.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4898.61474609375, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 10.203918942618898, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 10.206967192076284, + "BasicPerformanceMetrics.RECEIVED_DURATION": 4898.538818359375, + "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 10.203900640314695, + "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 23.1435546875, + "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 21.595458984375, "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, "BasicPerformanceMetrics.NUM_FRAMES_SENT": 50, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 18.10400390625, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 16.640625, - "BasicPerformanceMetrics.MAX_JITTER": 3.7236328125, - "BasicPerformanceMetrics.MIN_JITTER": 0.027587890625, - "BasicPerformanceMetrics.MEAN_JITTER": 1.1752726236979167, - "BasicPerformanceMetrics.STD_DEV_JITTER": 0.9299339011094103, - "CPUProfilingMetrics.MAX_CPU_UTIL": 2.5833333333333335, - "CPUProfilingMetrics.MIN_CPU_UTIL": 1.75, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 1.9683333333333333, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 0.31377628265303226, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 1.75 + "BasicPerformanceMetrics.MEAN_FRAME_RATE": 10.207125400865163, + "BasicPerformanceMetrics.MAX_JITTER": 4.39990234375, + "BasicPerformanceMetrics.MIN_JITTER": 0.003662109375, + "BasicPerformanceMetrics.MEAN_JITTER": 1.3919576009114583, + "BasicPerformanceMetrics.STD_DEV_JITTER": 1.1130362023020788, + "ResourceMetrics.BASELINE_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MAX_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MIN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.BASELINE_OVERALL_CPU_UTILIZATION": 8.333333333333334, + "ResourceMetrics.MEAN_OVERALL_CPU_UTILIZATION": 3.3837606837606846, + "ResourceMetrics.STDDEV_OVERALL_CPU_UTILIZATION": 4.676023109979668, + "ResourceMetrics.MAX_OVERALL_CPU_UTILIZATION": 24.916666666666668, + "ResourceMetrics.MIN_OVERALL_CPU_UTILIZATION": 0.0 }, "30.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4965.365234375, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 30.200860793349676, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 30.209258114902962, + "BasicPerformanceMetrics.RECEIVED_DURATION": 4964.967529296875, + "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 30.200929081641558, + "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 23.42578125, + "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 21.65869140625, "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, "BasicPerformanceMetrics.NUM_FRAMES_SENT": 150, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 17.947265625, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 16.566650390625, - "BasicPerformanceMetrics.MAX_JITTER": 4.0439453125, - "BasicPerformanceMetrics.MIN_JITTER": 0.01708984375, - "BasicPerformanceMetrics.MEAN_JITTER": 1.276980838260135, - "BasicPerformanceMetrics.STD_DEV_JITTER": 0.9746600428488554, - "CPUProfilingMetrics.MAX_CPU_UTIL": 5.408333333333334, - "CPUProfilingMetrics.MIN_CPU_UTIL": 4.916666666666667, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 5.173333333333334, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 0.1958882901610565, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 5.375 + "BasicPerformanceMetrics.MEAN_FRAME_RATE": 30.21167794449656, + "BasicPerformanceMetrics.MAX_JITTER": 4.967041015625, + "BasicPerformanceMetrics.MIN_JITTER": 0.06884765625, + "BasicPerformanceMetrics.MEAN_JITTER": 1.4820688608530406, + "BasicPerformanceMetrics.STD_DEV_JITTER": 1.171012397454802, + "ResourceMetrics.BASELINE_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MAX_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MIN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.BASELINE_OVERALL_CPU_UTILIZATION": 8.333333333333334, + "ResourceMetrics.MEAN_OVERALL_CPU_UTILIZATION": 8.698784722222221, + "ResourceMetrics.STDDEV_OVERALL_CPU_UTILIZATION": 6.412131669801993, + "ResourceMetrics.MAX_OVERALL_CPU_UTILIZATION": 34.666666666666664, + "ResourceMetrics.MIN_OVERALL_CPU_UTILIZATION": 0.0 }, "custom": { "data_resolution": "HD (1280,720)" }, "metadata": { "BenchmarkMetadata.NAME": "apriltag_ros AprilTagNode Benchmark", - "BenchmarkMetadata.TEST_FILE_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/scripts/apriltag_ros_apriltag_node.py", - "BenchmarkMetadata.TEST_DATETIME": "2023-05-25T06:44:00Z", - "BenchmarkMetadata.DEVICE_HOSTNAME": "ros-ljn7pgsgmvde5hrjrnikcaso7e", + "BenchmarkMetadata.TEST_FILE_PATH": "/opt/ros/humble/share/ros2_benchmark/scripts/apriltag_ros_apriltag_node.py", + "BenchmarkMetadata.TEST_DATETIME": "2024-12-06T14:26:23Z", + "BenchmarkMetadata.DEVICE_HOSTNAME": "bc5060b86ac24d55-5749c912f85749ed", "BenchmarkMetadata.DEVICE_ARCH": "aarch64", - "BenchmarkMetadata.DEVICE_OS": "Linux 5.10.104-tegra #1 SMP PREEMPT Sun Mar 19 07:55:28 PDT 2023", - "BenchmarkMetadata.IDLE_CPU_UTIL": 0.3333333333333333, + "BenchmarkMetadata.DEVICE_OS": "Linux 5.15.148-tegra #1 SMP PREEMPT Thu Sep 12 21:01:54 PDT 2024", + "BenchmarkMetadata.IDLE_CPU_UTIL": 0.9212962962962964, + "BenchmarkMetadata.IDLE_GPU_UTIL": 0.0, "BenchmarkMetadata.BENCHMARK_MODE": 1, - "BenchmarkMetadata.PEAK_THROUGHPUT_PREDICTION": 56.09375, - "BenchmarkMetadata.INPUT_DATA_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets/datasets/r2b_dataset/r2b_storage", + "BenchmarkMetadata.PEAK_THROUGHPUT_PREDICTION": 42.65625, + "BenchmarkMetadata.INPUT_DATA_PATH": "/workspaces/isaac_ros-dev/src/ros2_benchmark/assets/datasets/r2b_dataset/r2b_storage", "BenchmarkMetadata.INPUT_DATA_SIZE": 3087908864, "BenchmarkMetadata.INPUT_DATA_HASH": "5e8f11201fe10dbac7307a8628553e94", "BenchmarkMetadata.INPUT_DATA_START_TIME": 3.0, "BenchmarkMetadata.INPUT_DATA_END_TIME": 3.5, - "BenchmarkMetadata.CONFIG": "ros2_benchmark_config:\n additional_fixed_publisher_rate_tests:\n - 10.0\n - 30.0\n - 60.0\n assets_root: /workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets\n benchmark_duration: 5.0\n benchmark_mode: BenchmarkMode.LOOPING\n benchmark_name: apriltag_ros AprilTagNode Benchmark\n benchmark_namespace: r2b\n binary_search_acceptable_frame_loss_fraction: 0.05\n binary_search_acceptable_frame_rate_drop: 10.0\n binary_search_duration_fraction: 1.0\n binary_search_terminal_interval_width: 10.0\n cpu_profiling_interval_sec: 1.0\n custom_report_info:\n data_resolution: Resolution(1280, 720, HD)\n default_config_version: 0.30.0\n default_service_future_timeout_sec: 15.0\n enable_cpu_profiler: true\n enable_trial_buffer_preparation: false\n enforce_publisher_rate: false\n input_data_end_time: 3.5\n input_data_path: datasets/r2b_dataset/r2b_storage\n input_data_start_time: 3.0\n linear_scan_acceptable_frame_loss_fraction: 0.05\n linear_scan_acceptable_frame_rate_drop: 5.0\n linear_scan_duration_fraction: 1.0\n linear_scan_step_size: 5.0\n load_data_in_real_time: false\n log_file_name: kpi.json\n log_folder: /osmo/data/output\n monitor_info_list:\n - calculators:\n - class_name: BasicPerformanceCalculator\n config:\n report_prefix: ''\n module_name: ros2_benchmark.basic_performance_calculator\n service_name: start_monitoring0\n play_messages_service_future_timeout_sec: 15.0\n playback_message_buffer_size: 10\n publish_tf_messages_in_set_data: false\n publish_tf_static_messages_in_set_data: false\n publisher_lower_frequency: 10.0\n publisher_upper_frequency: 600.0\n record_data_timeline: false\n revise_timestamps_as_message_ids: false\n set_data_service_future_timeout_sec: 60.0\n setup_service_client_timeout_sec: 20.0\n start_monitoring_service_timeout_sec: 10\n start_recording_service_future_timeout_sec: 35.0\n start_recording_service_timeout_sec: 30\n test_iterations: 5\n" + "BenchmarkMetadata.CONFIG": "ros2_benchmark_config:\n additional_fixed_publisher_rate_tests:\n - 10.0\n - 30.0\n - 60.0\n assets_root: /workspaces/isaac_ros-dev/src/ros2_benchmark/assets\n benchmark_duration: 5.0\n benchmark_mode: BenchmarkMode.LOOPING\n benchmark_name: apriltag_ros AprilTagNode Benchmark\n benchmark_namespace: r2b\n binary_search_acceptable_frame_loss_fraction: 0.05\n binary_search_acceptable_frame_rate_drop: 10.0\n binary_search_duration_fraction: 1.0\n binary_search_terminal_interval_width: 10.0\n collect_start_timestamps_from_monitors: false\n custom_report_info:\n data_resolution: Resolution(1280, 720, HD)\n default_config_version: 0.30.0\n default_service_future_timeout_sec: 15.0\n enable_resource_profiler: true\n enable_trial_buffer_preparation: false\n enforce_publisher_rate: false\n export_monitor_raw_data: false\n input_data_end_time: 3.5\n input_data_path: datasets/r2b_dataset/r2b_storage\n input_data_start_time: 3.0\n linear_scan_acceptable_frame_loss_fraction: 0.05\n linear_scan_acceptable_frame_rate_drop: 5.0\n linear_scan_duration_fraction: 1.0\n linear_scan_step_size: 5.0\n load_data_in_real_time: false\n log_file_name: kpi.json\n log_folder: /osmo/data/output\n monitor_info_list:\n - calculators:\n - class_name: BasicPerformanceCalculator\n config:\n report_prefix: ''\n module_name: ros2_benchmark.basic_performance_calculator\n service_name: monitor_node0\n play_messages_service_future_timeout_sec: 15.0\n playback_message_buffer_size: 10\n post_trial_run_wait_time_sec: 0.0\n pre_stop_monitoring_wait_time_sec: 5.0\n pre_trial_run_wait_time_sec: 0.0\n publish_tf_messages_in_set_data: false\n publish_tf_static_messages_in_set_data: false\n publisher_lower_frequency: 10.0\n publisher_upper_frequency: 600.0\n record_data_timeline: false\n resource_profiling_interval_sec: 0.01\n revise_timestamps_as_message_ids: false\n ros1_ws: /workspaces/isaac_ros-dev/isaac_ros_1-dev\n set_data_service_future_timeout_sec: 60.0\n setup_service_client_timeout_sec: 20.0\n start_recording_service_future_timeout_sec: 35.0\n start_recording_service_timeout_sec: 30\n test_iterations: 5\n" } } \ No newline at end of file diff --git a/results/apriltag_ros_apriltag_node-agx_xavier.json b/results/apriltag_ros_apriltag_node-agx_xavier.json deleted file mode 100644 index 81a388f..0000000 --- a/results/apriltag_ros_apriltag_node-agx_xavier.json +++ /dev/null @@ -1,74 +0,0 @@ -{ - "BasicPerformanceMetrics.RECEIVED_DURATION": 4984.154541015625, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 37.8565936474434, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 36.38210610669438, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 6.666666666666667, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 188.0, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 27.948404947916668, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 46.131510416666664, - "BasicPerformanceMetrics.MAX_JITTER": 7.62451171875, - "BasicPerformanceMetrics.MIN_JITTER": 0.015380859375, - "BasicPerformanceMetrics.MEAN_JITTER": 1.402954783519553, - "BasicPerformanceMetrics.STD_DEV_JITTER": 1.0519826653879074, - "CPUProfilingMetrics.MAX_CPU_UTIL": 19.012500000000003, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.5, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 9.398750000000001, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 7.888288382593412, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 16.845833333333335, - "10.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4898.36328125, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 10.203860985547488, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 10.2074911820833, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 50, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 27.296142578125, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 25.553466796875, - "BasicPerformanceMetrics.MAX_JITTER": 6.6298828125, - "BasicPerformanceMetrics.MIN_JITTER": 0.08203125, - "BasicPerformanceMetrics.MEAN_JITTER": 2.3904571533203125, - "BasicPerformanceMetrics.STD_DEV_JITTER": 1.7451851150822548, - "CPUProfilingMetrics.MAX_CPU_UTIL": 6.35, - "CPUProfilingMetrics.MIN_CPU_UTIL": 4.9625, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 5.3999999999999995, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 0.5136268100479178, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 4.9625 - }, - "30.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4964.73046875, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 30.20019574091712, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 30.213120519665672, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 150, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 27.607421875, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 25.482666015625, - "BasicPerformanceMetrics.MAX_JITTER": 10.95263671875, - "BasicPerformanceMetrics.MIN_JITTER": 0.005615234375, - "BasicPerformanceMetrics.MEAN_JITTER": 2.2659912109375, - "BasicPerformanceMetrics.STD_DEV_JITTER": 1.7137552803028968, - "CPUProfilingMetrics.MAX_CPU_UTIL": 15.15, - "CPUProfilingMetrics.MIN_CPU_UTIL": 13.5375, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 14.367500000000001, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 0.6515654226553157, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 14.6 - }, - "custom": { - "data_resolution": "HD (1280,720)" - }, - "metadata": { - "BenchmarkMetadata.NAME": "apriltag_ros AprilTagNode Benchmark", - "BenchmarkMetadata.TEST_FILE_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/scripts/apriltag_ros_apriltag_node.py", - "BenchmarkMetadata.TEST_DATETIME": "2023-05-25T07:05:32Z", - "BenchmarkMetadata.DEVICE_HOSTNAME": "ros-qjkx4se23zb3fgy56r2a6ztpyi", - "BenchmarkMetadata.DEVICE_ARCH": "aarch64", - "BenchmarkMetadata.DEVICE_OS": "Linux 5.10.104-tegra #1 SMP PREEMPT Sun Mar 19 07:55:28 PDT 2023", - "BenchmarkMetadata.IDLE_CPU_UTIL": 2.5, - "BenchmarkMetadata.BENCHMARK_MODE": 1, - "BenchmarkMetadata.PEAK_THROUGHPUT_PREDICTION": 37.65625, - "BenchmarkMetadata.INPUT_DATA_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets/datasets/r2b_dataset/r2b_storage", - "BenchmarkMetadata.INPUT_DATA_SIZE": 3087908864, - "BenchmarkMetadata.INPUT_DATA_HASH": "5e8f11201fe10dbac7307a8628553e94", - "BenchmarkMetadata.INPUT_DATA_START_TIME": 3.0, - "BenchmarkMetadata.INPUT_DATA_END_TIME": 3.5, - "BenchmarkMetadata.CONFIG": "ros2_benchmark_config:\n additional_fixed_publisher_rate_tests:\n - 10.0\n - 30.0\n - 60.0\n assets_root: /workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets\n benchmark_duration: 5.0\n benchmark_mode: BenchmarkMode.LOOPING\n benchmark_name: apriltag_ros AprilTagNode Benchmark\n benchmark_namespace: r2b\n binary_search_acceptable_frame_loss_fraction: 0.05\n binary_search_acceptable_frame_rate_drop: 10.0\n binary_search_duration_fraction: 1.0\n binary_search_terminal_interval_width: 10.0\n cpu_profiling_interval_sec: 1.0\n custom_report_info:\n data_resolution: Resolution(1280, 720, HD)\n default_config_version: 0.30.0\n default_service_future_timeout_sec: 15.0\n enable_cpu_profiler: true\n enable_trial_buffer_preparation: false\n enforce_publisher_rate: false\n input_data_end_time: 3.5\n input_data_path: datasets/r2b_dataset/r2b_storage\n input_data_start_time: 3.0\n linear_scan_acceptable_frame_loss_fraction: 0.05\n linear_scan_acceptable_frame_rate_drop: 5.0\n linear_scan_duration_fraction: 1.0\n linear_scan_step_size: 5.0\n load_data_in_real_time: false\n log_file_name: kpi.json\n log_folder: /osmo/data/output\n monitor_info_list:\n - calculators:\n - class_name: BasicPerformanceCalculator\n config:\n report_prefix: ''\n module_name: ros2_benchmark.basic_performance_calculator\n service_name: start_monitoring0\n play_messages_service_future_timeout_sec: 15.0\n playback_message_buffer_size: 10\n publish_tf_messages_in_set_data: false\n publish_tf_static_messages_in_set_data: false\n publisher_lower_frequency: 10.0\n publisher_upper_frequency: 600.0\n record_data_timeline: false\n revise_timestamps_as_message_ids: false\n set_data_service_future_timeout_sec: 60.0\n setup_service_client_timeout_sec: 20.0\n start_monitoring_service_timeout_sec: 10\n start_recording_service_future_timeout_sec: 35.0\n start_recording_service_timeout_sec: 30\n test_iterations: 5\n" - } -} \ No newline at end of file diff --git a/results/apriltag_ros_apriltag_node-nuc_3060ti.json b/results/apriltag_ros_apriltag_node-nuc_3060ti.json deleted file mode 100644 index aa95350..0000000 --- a/results/apriltag_ros_apriltag_node-nuc_3060ti.json +++ /dev/null @@ -1,92 +0,0 @@ -{ - "BasicPerformanceMetrics.RECEIVED_DURATION": 4987.95166015625, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 98.16867257851332, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 94.48948749541258, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 17.666666666666668, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 489.0, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 11.159993489583334, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 18.058349609375, - "BasicPerformanceMetrics.MAX_JITTER": 7.860107421875, - "BasicPerformanceMetrics.MIN_JITTER": 0.000244140625, - "BasicPerformanceMetrics.MEAN_JITTER": 0.7944718389552039, - "BasicPerformanceMetrics.STD_DEV_JITTER": 0.8769630541471584, - "CPUProfilingMetrics.MAX_CPU_UTIL": 8.70625, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.125, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 3.7664772727272724, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 3.72173246127873, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 7.566666666666666, - "10.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4904.551025390625, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 10.203100995405165, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 10.194613072868933, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 50, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 11.37890625, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 15.458984375, - "BasicPerformanceMetrics.MAX_JITTER": 24.279052734375, - "BasicPerformanceMetrics.MIN_JITTER": 0.0068359375, - "BasicPerformanceMetrics.MEAN_JITTER": 9.678466796875, - "BasicPerformanceMetrics.STD_DEV_JITTER": 6.815641847103122, - "CPUProfilingMetrics.MAX_CPU_UTIL": 1.9375, - "CPUProfilingMetrics.MIN_CPU_UTIL": 1.44375, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 1.6612499999999997, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 0.19069773202636678, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 1.45 - }, - "60.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4983.95947265625, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 60.19939580147427, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 60.19310583200069, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 300, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 11.388427734375, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 11.9091796875, - "BasicPerformanceMetrics.MAX_JITTER": 16.4609375, - "BasicPerformanceMetrics.MIN_JITTER": 0.012939453125, - "BasicPerformanceMetrics.MEAN_JITTER": 4.056728286231124, - "BasicPerformanceMetrics.STD_DEV_JITTER": 3.3523021437093314, - "CPUProfilingMetrics.MAX_CPU_UTIL": 6.43125, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.3125, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 4.867708333333334, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 2.062504471796499, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 5.80625 - }, - "30.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4974.26611328125, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 30.20019574091712, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 30.155202111021204, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 150, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 11.394775390625, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 18.8056640625, - "BasicPerformanceMetrics.MAX_JITTER": 26.45361328125, - "BasicPerformanceMetrics.MIN_JITTER": 0.084716796875, - "BasicPerformanceMetrics.MEAN_JITTER": 9.300156052048141, - "BasicPerformanceMetrics.STD_DEV_JITTER": 7.040734789455976, - "CPUProfilingMetrics.MAX_CPU_UTIL": 4.706250000000001, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.25, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 3.6031250000000004, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 1.5261298324929635, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 4.18125 - }, - "custom": { - "data_resolution": "HD (1280,720)" - }, - "metadata": { - "BenchmarkMetadata.NAME": "apriltag_ros AprilTagNode Benchmark", - "BenchmarkMetadata.TEST_FILE_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/scripts/apriltag_ros_apriltag_node.py", - "BenchmarkMetadata.TEST_DATETIME": "2023-05-25T06:27:09Z", - "BenchmarkMetadata.DEVICE_HOSTNAME": "ros-oz6gfn6q35hanoqav5x5uyufii", - "BenchmarkMetadata.DEVICE_ARCH": "x86_64", - "BenchmarkMetadata.DEVICE_OS": "Linux 5.15.0-50-generic #56~20.04.1-Ubuntu SMP Tue Sep 27 15:51:29 UTC 2022", - "BenchmarkMetadata.IDLE_CPU_UTIL": 1.0, - "BenchmarkMetadata.BENCHMARK_MODE": 1, - "BenchmarkMetadata.PEAK_THROUGHPUT_PREDICTION": 97.96875, - "BenchmarkMetadata.INPUT_DATA_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets/datasets/r2b_dataset/r2b_storage", - "BenchmarkMetadata.INPUT_DATA_SIZE": 3087908864, - "BenchmarkMetadata.INPUT_DATA_HASH": "5e8f11201fe10dbac7307a8628553e94", - "BenchmarkMetadata.INPUT_DATA_START_TIME": 3.0, - "BenchmarkMetadata.INPUT_DATA_END_TIME": 3.5, - "BenchmarkMetadata.CONFIG": "ros2_benchmark_config:\n additional_fixed_publisher_rate_tests:\n - 10.0\n - 30.0\n - 60.0\n assets_root: /workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets\n benchmark_duration: 5.0\n benchmark_mode: BenchmarkMode.LOOPING\n benchmark_name: apriltag_ros AprilTagNode Benchmark\n benchmark_namespace: r2b\n binary_search_acceptable_frame_loss_fraction: 0.05\n binary_search_acceptable_frame_rate_drop: 10.0\n binary_search_duration_fraction: 1.0\n binary_search_terminal_interval_width: 10.0\n cpu_profiling_interval_sec: 1.0\n custom_report_info:\n data_resolution: Resolution(1280, 720, HD)\n default_config_version: 0.30.0\n default_service_future_timeout_sec: 15.0\n enable_cpu_profiler: true\n enable_trial_buffer_preparation: false\n enforce_publisher_rate: false\n input_data_end_time: 3.5\n input_data_path: datasets/r2b_dataset/r2b_storage\n input_data_start_time: 3.0\n linear_scan_acceptable_frame_loss_fraction: 0.05\n linear_scan_acceptable_frame_rate_drop: 5.0\n linear_scan_duration_fraction: 1.0\n linear_scan_step_size: 5.0\n load_data_in_real_time: false\n log_file_name: kpi.json\n log_folder: /osmo/data/output\n monitor_info_list:\n - calculators:\n - class_name: BasicPerformanceCalculator\n config:\n report_prefix: ''\n module_name: ros2_benchmark.basic_performance_calculator\n service_name: start_monitoring0\n play_messages_service_future_timeout_sec: 15.0\n playback_message_buffer_size: 10\n publish_tf_messages_in_set_data: false\n publish_tf_static_messages_in_set_data: false\n publisher_lower_frequency: 10.0\n publisher_upper_frequency: 600.0\n record_data_timeline: false\n revise_timestamps_as_message_ids: false\n set_data_service_future_timeout_sec: 60.0\n setup_service_client_timeout_sec: 20.0\n start_monitoring_service_timeout_sec: 10\n start_recording_service_future_timeout_sec: 35.0\n start_recording_service_timeout_sec: 30\n test_iterations: 5\n" - } -} \ No newline at end of file diff --git a/results/apriltag_ros_apriltag_node-nuc_4060ti.json b/results/apriltag_ros_apriltag_node-nuc_4060ti.json deleted file mode 100644 index 9e5ba89..0000000 --- a/results/apriltag_ros_apriltag_node-nuc_4060ti.json +++ /dev/null @@ -1,92 +0,0 @@ -{ - "BasicPerformanceMetrics.RECEIVED_DURATION": 4984.897379557292, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 93.16788729451925, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 90.82275426883587, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 11.0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 464.0, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 11.05078125, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 15.65625, - "BasicPerformanceMetrics.MAX_JITTER": 8.355224609375, - "BasicPerformanceMetrics.MIN_JITTER": 0.001220703125, - "BasicPerformanceMetrics.MEAN_JITTER": 1.098354026642184, - "BasicPerformanceMetrics.STD_DEV_JITTER": 1.2442838678829633, - "CPUProfilingMetrics.MAX_CPU_UTIL": 8.01875, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.125, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 3.664962121212121, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 3.5366901830238824, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 7.479166666666667, - "10.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4903.717529296875, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 10.203386677416095, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 10.196345874589825, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 50, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 10.755615234375, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 14.139404296875, - "BasicPerformanceMetrics.MAX_JITTER": 28.39697265625, - "BasicPerformanceMetrics.MIN_JITTER": 0.01611328125, - "BasicPerformanceMetrics.MEAN_JITTER": 11.692977905273438, - "BasicPerformanceMetrics.STD_DEV_JITTER": 8.73423272350045, - "CPUProfilingMetrics.MAX_CPU_UTIL": 2.30625, - "CPUProfilingMetrics.MIN_CPU_UTIL": 1.625, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 1.8175000000000001, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 0.25444179098567904, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 1.625 - }, - "60.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4986.884033203125, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 60.19968777290844, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 60.157805556049205, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 300, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 11.07373046875, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 14.543212890625, - "BasicPerformanceMetrics.MAX_JITTER": 17.064208984375, - "BasicPerformanceMetrics.MIN_JITTER": 0.0224609375, - "BasicPerformanceMetrics.MEAN_JITTER": 5.018775069473574, - "BasicPerformanceMetrics.STD_DEV_JITTER": 3.7673744801969997, - "CPUProfilingMetrics.MAX_CPU_UTIL": 6.949999999999999, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.25, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 5.547916666666667, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 2.3827304744533926, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 6.4625 - }, - "30.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4977.3671875, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 30.20018089633095, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 30.136414363140656, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 150, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 10.812744140625, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 21.322265625, - "BasicPerformanceMetrics.MAX_JITTER": 25.05419921875, - "BasicPerformanceMetrics.MIN_JITTER": 0.275390625, - "BasicPerformanceMetrics.MEAN_JITTER": 9.177018449113175, - "BasicPerformanceMetrics.STD_DEV_JITTER": 6.208451823735023, - "CPUProfilingMetrics.MAX_CPU_UTIL": 4.95625, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.375, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 3.4104166666666664, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 1.4304658817633116, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 3.775 - }, - "custom": { - "data_resolution": "HD (1280,720)" - }, - "metadata": { - "BenchmarkMetadata.NAME": "apriltag_ros AprilTagNode Benchmark", - "BenchmarkMetadata.TEST_FILE_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/scripts/apriltag_ros_apriltag_node.py", - "BenchmarkMetadata.TEST_DATETIME": "2023-05-25T08:08:50Z", - "BenchmarkMetadata.DEVICE_HOSTNAME": "ros-yvgkv733zfewzjlxrhgc5dhpai", - "BenchmarkMetadata.DEVICE_ARCH": "x86_64", - "BenchmarkMetadata.DEVICE_OS": "Linux 5.15.0-72-generic #79~20.04.1-Ubuntu SMP Thu Apr 20 22:12:07 UTC 2023", - "BenchmarkMetadata.IDLE_CPU_UTIL": 0.4375, - "BenchmarkMetadata.BENCHMARK_MODE": 1, - "BenchmarkMetadata.PEAK_THROUGHPUT_PREDICTION": 92.96875, - "BenchmarkMetadata.INPUT_DATA_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets/datasets/r2b_dataset/r2b_storage", - "BenchmarkMetadata.INPUT_DATA_SIZE": 3087908864, - "BenchmarkMetadata.INPUT_DATA_HASH": "5e8f11201fe10dbac7307a8628553e94", - "BenchmarkMetadata.INPUT_DATA_START_TIME": 3.0, - "BenchmarkMetadata.INPUT_DATA_END_TIME": 3.5, - "BenchmarkMetadata.CONFIG": "ros2_benchmark_config:\n additional_fixed_publisher_rate_tests:\n - 10.0\n - 30.0\n - 60.0\n assets_root: /workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets\n benchmark_duration: 5.0\n benchmark_mode: BenchmarkMode.LOOPING\n benchmark_name: apriltag_ros AprilTagNode Benchmark\n benchmark_namespace: r2b\n binary_search_acceptable_frame_loss_fraction: 0.05\n binary_search_acceptable_frame_rate_drop: 10.0\n binary_search_duration_fraction: 1.0\n binary_search_terminal_interval_width: 10.0\n cpu_profiling_interval_sec: 1.0\n custom_report_info:\n data_resolution: Resolution(1280, 720, HD)\n default_config_version: 0.30.0\n default_service_future_timeout_sec: 15.0\n enable_cpu_profiler: true\n enable_trial_buffer_preparation: false\n enforce_publisher_rate: false\n input_data_end_time: 3.5\n input_data_path: datasets/r2b_dataset/r2b_storage\n input_data_start_time: 3.0\n linear_scan_acceptable_frame_loss_fraction: 0.05\n linear_scan_acceptable_frame_rate_drop: 5.0\n linear_scan_duration_fraction: 1.0\n linear_scan_step_size: 5.0\n load_data_in_real_time: false\n log_file_name: kpi.json\n log_folder: /osmo/data/output\n monitor_info_list:\n - calculators:\n - class_name: BasicPerformanceCalculator\n config:\n report_prefix: ''\n module_name: ros2_benchmark.basic_performance_calculator\n service_name: start_monitoring0\n play_messages_service_future_timeout_sec: 15.0\n playback_message_buffer_size: 10\n publish_tf_messages_in_set_data: false\n publish_tf_static_messages_in_set_data: false\n publisher_lower_frequency: 10.0\n publisher_upper_frequency: 600.0\n record_data_timeline: false\n revise_timestamps_as_message_ids: false\n set_data_service_future_timeout_sec: 60.0\n setup_service_client_timeout_sec: 20.0\n start_monitoring_service_timeout_sec: 10\n start_recording_service_future_timeout_sec: 35.0\n start_recording_service_timeout_sec: 30\n test_iterations: 5\n" - } -} \ No newline at end of file diff --git a/results/apriltag_ros_apriltag_node-orin_nx.json b/results/apriltag_ros_apriltag_node-orin_nx.json index 1f9f7dc..085a9e9 100644 --- a/results/apriltag_ros_apriltag_node-orin_nx.json +++ b/results/apriltag_ros_apriltag_node-orin_nx.json @@ -1,74 +1,90 @@ { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4981.526448567708, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 52.07514475704795, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 51.389870728376344, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 3.0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 259.0, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 19.80322265625, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 27.807535807291668, - "BasicPerformanceMetrics.MAX_JITTER": 3.099609375, - "BasicPerformanceMetrics.MIN_JITTER": 0.003662109375, - "BasicPerformanceMetrics.MEAN_JITTER": 0.8361210860605315, - "BasicPerformanceMetrics.STD_DEV_JITTER": 0.709268795292024, - "CPUProfilingMetrics.MAX_CPU_UTIL": 16.25, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.125, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 7.9241666666666655, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 7.1801490865402995, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 15.120833333333332, + "BasicPerformanceMetrics.RECEIVED_DURATION": 4972.481201171875, + "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 56.293819102277126, + "BasicPerformanceMetrics.MEAN_FRAME_RATE": 56.17561873840202, + "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0.6666666666666666, + "BasicPerformanceMetrics.NUM_FRAMES_SENT": 280.0, + "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 18.614339192708332, + "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 17.179036458333332, + "BasicPerformanceMetrics.MAX_JITTER": 2.6943359375, + "BasicPerformanceMetrics.MIN_JITTER": 0.001220703125, + "BasicPerformanceMetrics.MEAN_JITTER": 0.7852581216276978, + "BasicPerformanceMetrics.STD_DEV_JITTER": 0.663964307608215, + "ResourceMetrics.BASELINE_OVERALL_CPU_UTILIZATION": 10.416666666666666, + "ResourceMetrics.MAX_OVERALL_CPU_UTILIZATION": 41.625, + "ResourceMetrics.MIN_OVERALL_CPU_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_OVERALL_CPU_UTILIZATION": 14.907856802317212, + "ResourceMetrics.STDDEV_OVERALL_CPU_UTILIZATION": 5.923149654796657, + "ResourceMetrics.BASELINE_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MAX_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MIN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_DEVICE_UTILIZATION": 0.0, "10.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4898.661865234375, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 10.203969274294005, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 10.206869013525546, + "BasicPerformanceMetrics.RECEIVED_DURATION": 4898.61767578125, + "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 10.203890980791723, + "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 18.27099609375, + "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 16.797119140625, "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, "BasicPerformanceMetrics.NUM_FRAMES_SENT": 50, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 19.772216796875, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 18.380126953125, - "BasicPerformanceMetrics.MAX_JITTER": 4.212890625, - "BasicPerformanceMetrics.MIN_JITTER": 0.075439453125, - "BasicPerformanceMetrics.MEAN_JITTER": 1.3140818277994792, - "BasicPerformanceMetrics.STD_DEV_JITTER": 1.0452383578045457, - "CPUProfilingMetrics.MAX_CPU_UTIL": 3.9875, - "CPUProfilingMetrics.MIN_CPU_UTIL": 2.7625, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 3.12, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 0.45494505162711685, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 3.15 + "BasicPerformanceMetrics.MEAN_FRAME_RATE": 10.206961087655367, + "BasicPerformanceMetrics.MAX_JITTER": 3.860107421875, + "BasicPerformanceMetrics.MIN_JITTER": 0.0185546875, + "BasicPerformanceMetrics.MEAN_JITTER": 1.1123809814453125, + "BasicPerformanceMetrics.STD_DEV_JITTER": 0.9683492766894772, + "ResourceMetrics.BASELINE_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MAX_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MIN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.BASELINE_OVERALL_CPU_UTILIZATION": 10.375, + "ResourceMetrics.MEAN_OVERALL_CPU_UTILIZATION": 3.4302884615384617, + "ResourceMetrics.STDDEV_OVERALL_CPU_UTILIZATION": 5.06869359149695, + "ResourceMetrics.MAX_OVERALL_CPU_UTILIZATION": 18.75, + "ResourceMetrics.MIN_OVERALL_CPU_UTILIZATION": 0.0 }, "30.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4964.620361328125, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 30.201870303907793, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 30.213790598859067, + "BasicPerformanceMetrics.RECEIVED_DURATION": 4965.427978515625, + "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 30.20111613376165, + "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 18.19677734375, + "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 16.9208984375, "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, "BasicPerformanceMetrics.NUM_FRAMES_SENT": 150, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 20.238525390625, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 18.279052734375, - "BasicPerformanceMetrics.MAX_JITTER": 4.612060546875, - "BasicPerformanceMetrics.MIN_JITTER": 0.0888671875, - "BasicPerformanceMetrics.MEAN_JITTER": 1.3350978542018581, - "BasicPerformanceMetrics.STD_DEV_JITTER": 1.0757268662918558, - "CPUProfilingMetrics.MAX_CPU_UTIL": 9.7875, - "CPUProfilingMetrics.MIN_CPU_UTIL": 8.1375, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 9.23, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 0.647765389628066, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 9.7625 + "BasicPerformanceMetrics.MEAN_FRAME_RATE": 30.208876384677982, + "BasicPerformanceMetrics.MAX_JITTER": 4.295654296875, + "BasicPerformanceMetrics.MIN_JITTER": 0.002685546875, + "BasicPerformanceMetrics.MEAN_JITTER": 1.140380859375, + "BasicPerformanceMetrics.STD_DEV_JITTER": 0.9142497336413162, + "ResourceMetrics.BASELINE_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MAX_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MIN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.BASELINE_OVERALL_CPU_UTILIZATION": 6.25, + "ResourceMetrics.MEAN_OVERALL_CPU_UTILIZATION": 7.564024390243903, + "ResourceMetrics.STDDEV_OVERALL_CPU_UTILIZATION": 5.6220196678548335, + "ResourceMetrics.MAX_OVERALL_CPU_UTILIZATION": 18.75, + "ResourceMetrics.MIN_OVERALL_CPU_UTILIZATION": 0.0 }, "custom": { "data_resolution": "HD (1280,720)" }, "metadata": { "BenchmarkMetadata.NAME": "apriltag_ros AprilTagNode Benchmark", - "BenchmarkMetadata.TEST_FILE_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/scripts/apriltag_ros_apriltag_node.py", - "BenchmarkMetadata.TEST_DATETIME": "2023-05-25T07:06:17Z", - "BenchmarkMetadata.DEVICE_HOSTNAME": "ros-3rsggxeuqvc73aavb2ftxfwc5m", + "BenchmarkMetadata.TEST_FILE_PATH": "/opt/ros/humble/share/ros2_benchmark/scripts/apriltag_ros_apriltag_node.py", + "BenchmarkMetadata.TEST_DATETIME": "2024-12-06T13:22:25Z", + "BenchmarkMetadata.DEVICE_HOSTNAME": "e40575214ae14d83-c1bbb2e4dd854cd5", "BenchmarkMetadata.DEVICE_ARCH": "aarch64", - "BenchmarkMetadata.DEVICE_OS": "Linux 5.10.104-tegra #1 SMP PREEMPT Wed Mar 15 00:17:07 PDT 2023", - "BenchmarkMetadata.IDLE_CPU_UTIL": 0.375, + "BenchmarkMetadata.DEVICE_OS": "Linux 5.15.148-tegra #1 SMP PREEMPT Thu Sep 12 21:01:54 PDT 2024", + "BenchmarkMetadata.IDLE_CPU_UTIL": 1.0069444444444444, + "BenchmarkMetadata.IDLE_GPU_UTIL": 0.0, "BenchmarkMetadata.BENCHMARK_MODE": 1, - "BenchmarkMetadata.PEAK_THROUGHPUT_PREDICTION": 51.875, - "BenchmarkMetadata.INPUT_DATA_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets/datasets/r2b_dataset/r2b_storage", + "BenchmarkMetadata.PEAK_THROUGHPUT_PREDICTION": 56.09375, + "BenchmarkMetadata.INPUT_DATA_PATH": "/workspaces/isaac_ros-dev/src/ros2_benchmark/assets/datasets/r2b_dataset/r2b_storage", "BenchmarkMetadata.INPUT_DATA_SIZE": 3087908864, "BenchmarkMetadata.INPUT_DATA_HASH": "5e8f11201fe10dbac7307a8628553e94", "BenchmarkMetadata.INPUT_DATA_START_TIME": 3.0, "BenchmarkMetadata.INPUT_DATA_END_TIME": 3.5, - "BenchmarkMetadata.CONFIG": "ros2_benchmark_config:\n additional_fixed_publisher_rate_tests:\n - 10.0\n - 30.0\n - 60.0\n assets_root: /workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets\n benchmark_duration: 5.0\n benchmark_mode: BenchmarkMode.LOOPING\n benchmark_name: apriltag_ros AprilTagNode Benchmark\n benchmark_namespace: r2b\n binary_search_acceptable_frame_loss_fraction: 0.05\n binary_search_acceptable_frame_rate_drop: 10.0\n binary_search_duration_fraction: 1.0\n binary_search_terminal_interval_width: 10.0\n cpu_profiling_interval_sec: 1.0\n custom_report_info:\n data_resolution: Resolution(1280, 720, HD)\n default_config_version: 0.30.0\n default_service_future_timeout_sec: 15.0\n enable_cpu_profiler: true\n enable_trial_buffer_preparation: false\n enforce_publisher_rate: false\n input_data_end_time: 3.5\n input_data_path: datasets/r2b_dataset/r2b_storage\n input_data_start_time: 3.0\n linear_scan_acceptable_frame_loss_fraction: 0.05\n linear_scan_acceptable_frame_rate_drop: 5.0\n linear_scan_duration_fraction: 1.0\n linear_scan_step_size: 5.0\n load_data_in_real_time: false\n log_file_name: kpi.json\n log_folder: /osmo/data/output\n monitor_info_list:\n - calculators:\n - class_name: BasicPerformanceCalculator\n config:\n report_prefix: ''\n module_name: ros2_benchmark.basic_performance_calculator\n service_name: start_monitoring0\n play_messages_service_future_timeout_sec: 15.0\n playback_message_buffer_size: 10\n publish_tf_messages_in_set_data: false\n publish_tf_static_messages_in_set_data: false\n publisher_lower_frequency: 10.0\n publisher_upper_frequency: 600.0\n record_data_timeline: false\n revise_timestamps_as_message_ids: false\n set_data_service_future_timeout_sec: 60.0\n setup_service_client_timeout_sec: 20.0\n start_monitoring_service_timeout_sec: 10\n start_recording_service_future_timeout_sec: 35.0\n start_recording_service_timeout_sec: 30\n test_iterations: 5\n" + "BenchmarkMetadata.CONFIG": "ros2_benchmark_config:\n additional_fixed_publisher_rate_tests:\n - 10.0\n - 30.0\n - 60.0\n assets_root: /workspaces/isaac_ros-dev/src/ros2_benchmark/assets\n benchmark_duration: 5.0\n benchmark_mode: BenchmarkMode.LOOPING\n benchmark_name: apriltag_ros AprilTagNode Benchmark\n benchmark_namespace: r2b\n binary_search_acceptable_frame_loss_fraction: 0.05\n binary_search_acceptable_frame_rate_drop: 10.0\n binary_search_duration_fraction: 1.0\n binary_search_terminal_interval_width: 10.0\n collect_start_timestamps_from_monitors: false\n custom_report_info:\n data_resolution: Resolution(1280, 720, HD)\n default_config_version: 0.30.0\n default_service_future_timeout_sec: 15.0\n enable_resource_profiler: true\n enable_trial_buffer_preparation: false\n enforce_publisher_rate: false\n export_monitor_raw_data: false\n input_data_end_time: 3.5\n input_data_path: datasets/r2b_dataset/r2b_storage\n input_data_start_time: 3.0\n linear_scan_acceptable_frame_loss_fraction: 0.05\n linear_scan_acceptable_frame_rate_drop: 5.0\n linear_scan_duration_fraction: 1.0\n linear_scan_step_size: 5.0\n load_data_in_real_time: false\n log_file_name: kpi.json\n log_folder: /osmo/data/output\n monitor_info_list:\n - calculators:\n - class_name: BasicPerformanceCalculator\n config:\n report_prefix: ''\n module_name: ros2_benchmark.basic_performance_calculator\n service_name: monitor_node0\n play_messages_service_future_timeout_sec: 15.0\n playback_message_buffer_size: 10\n post_trial_run_wait_time_sec: 0.0\n pre_stop_monitoring_wait_time_sec: 5.0\n pre_trial_run_wait_time_sec: 0.0\n publish_tf_messages_in_set_data: false\n publish_tf_static_messages_in_set_data: false\n publisher_lower_frequency: 10.0\n publisher_upper_frequency: 600.0\n record_data_timeline: false\n resource_profiling_interval_sec: 0.01\n revise_timestamps_as_message_ids: false\n ros1_ws: /workspaces/isaac_ros-dev/isaac_ros_1-dev\n set_data_service_future_timeout_sec: 60.0\n setup_service_client_timeout_sec: 20.0\n start_recording_service_future_timeout_sec: 35.0\n start_recording_service_timeout_sec: 30\n test_iterations: 5\n" } } \ No newline at end of file diff --git a/results/apriltag_ros_apriltag_node-x86-4090.json b/results/apriltag_ros_apriltag_node-x86-4090.json new file mode 100644 index 0000000..fb23ebd --- /dev/null +++ b/results/apriltag_ros_apriltag_node-x86-4090.json @@ -0,0 +1,133 @@ +{ + "BasicPerformanceMetrics.RECEIVED_DURATION": 4995.008219401042, + "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 130.03199622418126, + "BasicPerformanceMetrics.MEAN_FRAME_RATE": 127.07254663528431, + "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 14.333333333333334, + "BasicPerformanceMetrics.NUM_FRAMES_SENT": 649.0, + "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 8.263590494791666, + "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 12.297281901041666, + "BasicPerformanceMetrics.MAX_JITTER": 1.662109375, + "BasicPerformanceMetrics.MIN_JITTER": 0.000732421875, + "BasicPerformanceMetrics.MEAN_JITTER": 0.3691437153876582, + "BasicPerformanceMetrics.STD_DEV_JITTER": 0.28380258820119353, + "ResourceMetrics.BASELINE_OVERALL_CPU_UTILIZATION": 4.166666666666667, + "ResourceMetrics.MAX_OVERALL_CPU_UTILIZATION": 12.5, + "ResourceMetrics.MIN_OVERALL_CPU_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_OVERALL_CPU_UTILIZATION": 1.574028904916804, + "ResourceMetrics.STDDEV_OVERALL_CPU_UTILIZATION": 2.2742218527813645, + "ResourceMetrics.BASELINE_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MAX_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MIN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.BASELINE_DEVICE_MEMORY_UTILIZATION": 1.9400000000000002, + "ResourceMetrics.MAX_DEVICE_MEMORY_UTILIZATION": 1.94, + "ResourceMetrics.MIN_DEVICE_MEMORY_UTILIZATION": 1.94, + "ResourceMetrics.MEAN_DEVICE_MEMORY_UTILIZATION": 1.9400000000000002, + "ResourceMetrics.STDDEV_DEVICE_MEMORY_UTILIZATION": 2.9605947323337506e-16, + "10.0fps": { + "BasicPerformanceMetrics.RECEIVED_DURATION": 4899.661376953125, + "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 10.203959614641088, + "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 8.41455078125, + "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 8.017333984375, + "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, + "BasicPerformanceMetrics.NUM_FRAMES_SENT": 50, + "BasicPerformanceMetrics.MEAN_FRAME_RATE": 10.20478685224829, + "BasicPerformanceMetrics.MAX_JITTER": 1.51806640625, + "BasicPerformanceMetrics.MIN_JITTER": 0.042724609375, + "BasicPerformanceMetrics.MEAN_JITTER": 0.5471293131510416, + "BasicPerformanceMetrics.STD_DEV_JITTER": 0.4076402118723626, + "ResourceMetrics.BASELINE_OVERALL_CPU_UTILIZATION": 4.6875, + "ResourceMetrics.MEAN_OVERALL_CPU_UTILIZATION": 0.19112124463519314, + "ResourceMetrics.MAX_OVERALL_CPU_UTILIZATION": 14.0625, + "ResourceMetrics.MIN_OVERALL_CPU_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_OVERALL_CPU_UTILIZATION": 0.9438156172222486, + "ResourceMetrics.BASELINE_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MAX_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MIN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.BASELINE_DEVICE_MEMORY_UTILIZATION": 1.94, + "ResourceMetrics.MEAN_DEVICE_MEMORY_UTILIZATION": 1.9400000000000002, + "ResourceMetrics.STDDEV_DEVICE_MEMORY_UTILIZATION": 2.220446049250313e-16, + "ResourceMetrics.MAX_DEVICE_MEMORY_UTILIZATION": 1.94, + "ResourceMetrics.MIN_DEVICE_MEMORY_UTILIZATION": 1.94 + }, + "60.0fps": { + "BasicPerformanceMetrics.RECEIVED_DURATION": 4982.72265625, + "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 60.20007707255961, + "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 8.2724609375, + "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 7.61279296875, + "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, + "BasicPerformanceMetrics.NUM_FRAMES_SENT": 300, + "BasicPerformanceMetrics.MEAN_FRAME_RATE": 60.2080470249934, + "BasicPerformanceMetrics.MAX_JITTER": 1.7421875, + "BasicPerformanceMetrics.MIN_JITTER": 0.00146484375, + "BasicPerformanceMetrics.MEAN_JITTER": 0.5110682545092282, + "BasicPerformanceMetrics.STD_DEV_JITTER": 0.3761377028194193, + "ResourceMetrics.BASELINE_OVERALL_CPU_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_OVERALL_CPU_UTILIZATION": 0.8531852248394004, + "ResourceMetrics.MAX_OVERALL_CPU_UTILIZATION": 7.8125, + "ResourceMetrics.MIN_OVERALL_CPU_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_OVERALL_CPU_UTILIZATION": 1.4564523927072963, + "ResourceMetrics.BASELINE_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MAX_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MIN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.BASELINE_DEVICE_MEMORY_UTILIZATION": 1.94, + "ResourceMetrics.MEAN_DEVICE_MEMORY_UTILIZATION": 1.9400000000000004, + "ResourceMetrics.STDDEV_DEVICE_MEMORY_UTILIZATION": 4.440892098500626e-16, + "ResourceMetrics.MAX_DEVICE_MEMORY_UTILIZATION": 1.94, + "ResourceMetrics.MIN_DEVICE_MEMORY_UTILIZATION": 1.94 + }, + "30.0fps": { + "BasicPerformanceMetrics.RECEIVED_DURATION": 4966.6220703125, + "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 30.200988463015953, + "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 8.24169921875, + "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 8.138916015625, + "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, + "BasicPerformanceMetrics.NUM_FRAMES_SENT": 150, + "BasicPerformanceMetrics.MEAN_FRAME_RATE": 30.201613466144405, + "BasicPerformanceMetrics.MAX_JITTER": 1.916748046875, + "BasicPerformanceMetrics.MIN_JITTER": 0.004150390625, + "BasicPerformanceMetrics.MEAN_JITTER": 0.5555584881756757, + "BasicPerformanceMetrics.STD_DEV_JITTER": 0.3958971910643474, + "ResourceMetrics.BASELINE_OVERALL_CPU_UTILIZATION": 3.125, + "ResourceMetrics.MEAN_OVERALL_CPU_UTILIZATION": 0.4416488222698073, + "ResourceMetrics.MAX_OVERALL_CPU_UTILIZATION": 14.0625, + "ResourceMetrics.MIN_OVERALL_CPU_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_OVERALL_CPU_UTILIZATION": 1.3807759688058754, + "ResourceMetrics.BASELINE_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MAX_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MIN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.BASELINE_DEVICE_MEMORY_UTILIZATION": 1.94, + "ResourceMetrics.MEAN_DEVICE_MEMORY_UTILIZATION": 1.9400000000000004, + "ResourceMetrics.STDDEV_DEVICE_MEMORY_UTILIZATION": 4.440892098500626e-16, + "ResourceMetrics.MAX_DEVICE_MEMORY_UTILIZATION": 1.94, + "ResourceMetrics.MIN_DEVICE_MEMORY_UTILIZATION": 1.94 + }, + "custom": { + "data_resolution": "HD (1280,720)" + }, + "metadata": { + "BenchmarkMetadata.NAME": "apriltag_ros AprilTagNode Benchmark", + "BenchmarkMetadata.TEST_FILE_PATH": "/opt/ros/humble/share/ros2_benchmark/scripts/apriltag_ros_apriltag_node.py", + "BenchmarkMetadata.TEST_DATETIME": "2024-12-06T19:47:46Z", + "BenchmarkMetadata.DEVICE_HOSTNAME": "ecb2742e261a4e56-bad5639dcec748d3", + "BenchmarkMetadata.DEVICE_ARCH": "x86_64", + "BenchmarkMetadata.DEVICE_OS": "Linux 6.8.0-48-generic #48~22.04.1-Ubuntu SMP PREEMPT_DYNAMIC Mon Oct 7 11:24:13 UTC 2", + "BenchmarkMetadata.IDLE_CPU_UTIL": 0.0, + "BenchmarkMetadata.IDLE_GPU_UTIL": 0, + "BenchmarkMetadata.BENCHMARK_MODE": 1, + "BenchmarkMetadata.PEAK_THROUGHPUT_PREDICTION": 129.84375, + "BenchmarkMetadata.INPUT_DATA_PATH": "/workspaces/isaac_ros-dev/src/ros2_benchmark/assets/datasets/r2b_dataset/r2b_storage", + "BenchmarkMetadata.INPUT_DATA_SIZE": 3087908864, + "BenchmarkMetadata.INPUT_DATA_HASH": "5e8f11201fe10dbac7307a8628553e94", + "BenchmarkMetadata.INPUT_DATA_START_TIME": 3.0, + "BenchmarkMetadata.INPUT_DATA_END_TIME": 3.5, + "BenchmarkMetadata.CONFIG": "ros2_benchmark_config:\n additional_fixed_publisher_rate_tests:\n - 10.0\n - 30.0\n - 60.0\n assets_root: /workspaces/isaac_ros-dev/src/ros2_benchmark/assets\n benchmark_duration: 5.0\n benchmark_mode: BenchmarkMode.LOOPING\n benchmark_name: apriltag_ros AprilTagNode Benchmark\n benchmark_namespace: r2b\n binary_search_acceptable_frame_loss_fraction: 0.05\n binary_search_acceptable_frame_rate_drop: 10.0\n binary_search_duration_fraction: 1.0\n binary_search_terminal_interval_width: 10.0\n collect_start_timestamps_from_monitors: false\n custom_report_info:\n data_resolution: Resolution(1280, 720, HD)\n default_config_version: 0.30.0\n default_service_future_timeout_sec: 15.0\n enable_resource_profiler: true\n enable_trial_buffer_preparation: false\n enforce_publisher_rate: false\n export_monitor_raw_data: false\n input_data_end_time: 3.5\n input_data_path: datasets/r2b_dataset/r2b_storage\n input_data_start_time: 3.0\n linear_scan_acceptable_frame_loss_fraction: 0.05\n linear_scan_acceptable_frame_rate_drop: 5.0\n linear_scan_duration_fraction: 1.0\n linear_scan_step_size: 5.0\n load_data_in_real_time: false\n log_file_name: kpi.json\n log_folder: /osmo/data/output\n monitor_info_list:\n - calculators:\n - class_name: BasicPerformanceCalculator\n config:\n report_prefix: ''\n module_name: ros2_benchmark.basic_performance_calculator\n service_name: monitor_node0\n play_messages_service_future_timeout_sec: 15.0\n playback_message_buffer_size: 10\n post_trial_run_wait_time_sec: 0.0\n pre_stop_monitoring_wait_time_sec: 5.0\n pre_trial_run_wait_time_sec: 0.0\n publish_tf_messages_in_set_data: false\n publish_tf_static_messages_in_set_data: false\n publisher_lower_frequency: 10.0\n publisher_upper_frequency: 600.0\n record_data_timeline: false\n resource_profiling_interval_sec: 0.01\n revise_timestamps_as_message_ids: false\n ros1_ws: /workspaces/isaac_ros-dev/isaac_ros_1-dev\n set_data_service_future_timeout_sec: 60.0\n setup_service_client_timeout_sec: 20.0\n start_recording_service_future_timeout_sec: 35.0\n start_recording_service_timeout_sec: 30\n test_iterations: 5\n" + } +} \ No newline at end of file diff --git a/results/apriltag_ros_apriltag_node-x86_4090.json b/results/apriltag_ros_apriltag_node-x86_4090.json deleted file mode 100644 index 50fbb19..0000000 --- a/results/apriltag_ros_apriltag_node-x86_4090.json +++ /dev/null @@ -1,92 +0,0 @@ -{ - "BasicPerformanceMetrics.RECEIVED_DURATION": 4984.948649088542, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 102.38633845038343, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 100.17473419326211, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 10.666666666666666, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 510.0, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 10.26708984375, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 14.072672526041666, - "BasicPerformanceMetrics.MAX_JITTER": 9.23828125, - "BasicPerformanceMetrics.MIN_JITTER": 0.00048828125, - "BasicPerformanceMetrics.MEAN_JITTER": 0.6934538810483871, - "BasicPerformanceMetrics.STD_DEV_JITTER": 0.804873258338719, - "CPUProfilingMetrics.MAX_CPU_UTIL": 8.993749999999999, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.0625, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 3.917045454545455, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 3.817219551965696, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 7.804166666666666, - "10.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4923.239013671875, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 10.202912413377373, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 10.155915620011458, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 50, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 10.1708984375, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 32.848388671875, - "BasicPerformanceMetrics.MAX_JITTER": 39.037109375, - "BasicPerformanceMetrics.MIN_JITTER": 0.0546875, - "BasicPerformanceMetrics.MEAN_JITTER": 17.362279256184895, - "BasicPerformanceMetrics.STD_DEV_JITTER": 11.102397545717757, - "CPUProfilingMetrics.MAX_CPU_UTIL": 2.875, - "CPUProfilingMetrics.MIN_CPU_UTIL": 1.95, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 2.26125, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 0.31879656522616423, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 1.95 - }, - "60.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4988.5185546875, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 60.1981306246526, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 60.13809444852173, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 300, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 10.21533203125, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 15.1904296875, - "BasicPerformanceMetrics.MAX_JITTER": 20.7265625, - "BasicPerformanceMetrics.MIN_JITTER": 0.00244140625, - "BasicPerformanceMetrics.MEAN_JITTER": 5.277850055054531, - "BasicPerformanceMetrics.STD_DEV_JITTER": 4.286050484475102, - "CPUProfilingMetrics.MAX_CPU_UTIL": 6.74375, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.4375, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 5.534375, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 2.285340938049507, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 6.38125 - }, - "30.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4974.599365234375, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 30.199460951418683, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 30.153181992562903, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 150, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 10.20263671875, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 17.825927734375, - "BasicPerformanceMetrics.MAX_JITTER": 42.45751953125, - "BasicPerformanceMetrics.MIN_JITTER": 0.1015625, - "BasicPerformanceMetrics.MEAN_JITTER": 18.260954985747468, - "BasicPerformanceMetrics.STD_DEV_JITTER": 12.702126228622392, - "CPUProfilingMetrics.MAX_CPU_UTIL": 5.300000000000001, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.3125, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 3.9354166666666663, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 1.6732370490227089, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 4.8812500000000005 - }, - "custom": { - "data_resolution": "HD (1280,720)" - }, - "metadata": { - "BenchmarkMetadata.NAME": "apriltag_ros AprilTagNode Benchmark", - "BenchmarkMetadata.TEST_FILE_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/scripts/apriltag_ros_apriltag_node.py", - "BenchmarkMetadata.TEST_DATETIME": "2023-05-25T06:30:32Z", - "BenchmarkMetadata.DEVICE_HOSTNAME": "ros-brotdg2w3fhz3fngg5b6hac6ea", - "BenchmarkMetadata.DEVICE_ARCH": "x86_64", - "BenchmarkMetadata.DEVICE_OS": "Linux 5.4.0-131-generic #147-Ubuntu SMP Fri Oct 14 17:07:22 UTC 2022", - "BenchmarkMetadata.IDLE_CPU_UTIL": 0.125, - "BenchmarkMetadata.BENCHMARK_MODE": 1, - "BenchmarkMetadata.PEAK_THROUGHPUT_PREDICTION": 102.1875, - "BenchmarkMetadata.INPUT_DATA_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets/datasets/r2b_dataset/r2b_storage", - "BenchmarkMetadata.INPUT_DATA_SIZE": 3087908864, - "BenchmarkMetadata.INPUT_DATA_HASH": "5e8f11201fe10dbac7307a8628553e94", - "BenchmarkMetadata.INPUT_DATA_START_TIME": 3.0, - "BenchmarkMetadata.INPUT_DATA_END_TIME": 3.5, - "BenchmarkMetadata.CONFIG": "ros2_benchmark_config:\n additional_fixed_publisher_rate_tests:\n - 10.0\n - 30.0\n - 60.0\n assets_root: /workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets\n benchmark_duration: 5.0\n benchmark_mode: BenchmarkMode.LOOPING\n benchmark_name: apriltag_ros AprilTagNode Benchmark\n benchmark_namespace: r2b\n binary_search_acceptable_frame_loss_fraction: 0.05\n binary_search_acceptable_frame_rate_drop: 10.0\n binary_search_duration_fraction: 1.0\n binary_search_terminal_interval_width: 10.0\n cpu_profiling_interval_sec: 1.0\n custom_report_info:\n data_resolution: Resolution(1280, 720, HD)\n default_config_version: 0.30.0\n default_service_future_timeout_sec: 15.0\n enable_cpu_profiler: true\n enable_trial_buffer_preparation: false\n enforce_publisher_rate: false\n input_data_end_time: 3.5\n input_data_path: datasets/r2b_dataset/r2b_storage\n input_data_start_time: 3.0\n linear_scan_acceptable_frame_loss_fraction: 0.05\n linear_scan_acceptable_frame_rate_drop: 5.0\n linear_scan_duration_fraction: 1.0\n linear_scan_step_size: 5.0\n load_data_in_real_time: false\n log_file_name: kpi.json\n log_folder: /osmo/data/output\n monitor_info_list:\n - calculators:\n - class_name: BasicPerformanceCalculator\n config:\n report_prefix: ''\n module_name: ros2_benchmark.basic_performance_calculator\n service_name: start_monitoring0\n play_messages_service_future_timeout_sec: 15.0\n playback_message_buffer_size: 10\n publish_tf_messages_in_set_data: false\n publish_tf_static_messages_in_set_data: false\n publisher_lower_frequency: 10.0\n publisher_upper_frequency: 600.0\n record_data_timeline: false\n revise_timestamps_as_message_ids: false\n set_data_service_future_timeout_sec: 60.0\n setup_service_client_timeout_sec: 20.0\n start_monitoring_service_timeout_sec: 10\n start_recording_service_future_timeout_sec: 35.0\n start_recording_service_timeout_sec: 30\n test_iterations: 5\n" - } -} \ No newline at end of file diff --git a/results/image_proc_rectify_node-agx_orin.json b/results/image_proc_rectify_node-agx_orin.json index 5fc77aa..661c718 100644 --- a/results/image_proc_rectify_node-agx_orin.json +++ b/results/image_proc_rectify_node-agx_orin.json @@ -1,90 +1,111 @@ { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4992.956624348958, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 185.27436528077706, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 185.2487567630416, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0.0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 925.0, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 5.925618489583333, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 6.235514322916667, - "BasicPerformanceMetrics.MAX_JITTER": 2.10546875, + "BasicPerformanceMetrics.RECEIVED_DURATION": 4993.575113932292, + "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 161.09617153572077, + "BasicPerformanceMetrics.MEAN_FRAME_RATE": 157.38247598840238, + "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 18.0, + "BasicPerformanceMetrics.NUM_FRAMES_SENT": 804.0, + "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 6.680501302083333, + "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 9.546061197916666, + "BasicPerformanceMetrics.MAX_JITTER": 3.31591796875, "BasicPerformanceMetrics.MIN_JITTER": 0.000244140625, - "BasicPerformanceMetrics.MEAN_JITTER": 0.17703528109764355, - "BasicPerformanceMetrics.STD_DEV_JITTER": 0.1799133945593814, - "CPUProfilingMetrics.MAX_CPU_UTIL": 50.116666666666674, - "CPUProfilingMetrics.MIN_CPU_UTIL": 47.175000000000004, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 49.080000000000005, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 0.7629188764895664, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 48.583333333333336, + "BasicPerformanceMetrics.MEAN_JITTER": 0.7009439273756377, + "BasicPerformanceMetrics.STD_DEV_JITTER": 0.7443036937752062, + "ResourceMetrics.BASELINE_OVERALL_CPU_UTILIZATION": 45.22222222222223, + "ResourceMetrics.MAX_OVERALL_CPU_UTILIZATION": 77.66666666666667, + "ResourceMetrics.MIN_OVERALL_CPU_UTILIZATION": 12.5, + "ResourceMetrics.MEAN_OVERALL_CPU_UTILIZATION": 45.64520202020202, + "ResourceMetrics.STDDEV_OVERALL_CPU_UTILIZATION": 12.329995697450668, + "ResourceMetrics.BASELINE_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MAX_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MIN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_DEVICE_UTILIZATION": 0.0, "10.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4899.681884765625, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 10.20390165710765, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 10.204744139708925, + "BasicPerformanceMetrics.RECEIVED_DURATION": 4899.9345703125, + "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 10.203975883540748, + "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 6.533447265625, + "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 6.417236328125, "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, "BasicPerformanceMetrics.NUM_FRAMES_SENT": 50, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 5.76513671875, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 5.360595703125, - "BasicPerformanceMetrics.MAX_JITTER": 0.76171875, - "BasicPerformanceMetrics.MIN_JITTER": 0.00390625, - "BasicPerformanceMetrics.MEAN_JITTER": 0.217132568359375, - "BasicPerformanceMetrics.STD_DEV_JITTER": 0.18971660569430185, - "CPUProfilingMetrics.MAX_CPU_UTIL": 4.7749999999999995, - "CPUProfilingMetrics.MIN_CPU_UTIL": 3.7166666666666663, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 3.9816666666666665, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 0.40597618977800476, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 3.7416666666666667 + "BasicPerformanceMetrics.MEAN_FRAME_RATE": 10.204217889548508, + "BasicPerformanceMetrics.MAX_JITTER": 3.037841796875, + "BasicPerformanceMetrics.MIN_JITTER": 0.001708984375, + "BasicPerformanceMetrics.MEAN_JITTER": 0.8691558837890625, + "BasicPerformanceMetrics.STD_DEV_JITTER": 0.9048978216355874, + "ResourceMetrics.BASELINE_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MAX_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MIN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.BASELINE_OVERALL_CPU_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_OVERALL_CPU_UTILIZATION": 3.7466856060606064, + "ResourceMetrics.STDDEV_OVERALL_CPU_UTILIZATION": 7.781052164716886, + "ResourceMetrics.MAX_OVERALL_CPU_UTILIZATION": 36.083333333333336, + "ResourceMetrics.MIN_OVERALL_CPU_UTILIZATION": 0.0 }, "60.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4983.13671875, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 60.19983523430513, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 60.20304417319977, + "BasicPerformanceMetrics.RECEIVED_DURATION": 4982.437255859375, + "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 60.20200300020367, + "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 7.078857421875, + "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 6.293212890625, "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, "BasicPerformanceMetrics.NUM_FRAMES_SENT": 300, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 5.60498046875, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 5.33935546875, - "BasicPerformanceMetrics.MAX_JITTER": 4.03125, - "BasicPerformanceMetrics.MIN_JITTER": 0.002197265625, - "BasicPerformanceMetrics.MEAN_JITTER": 0.25495163066275167, - "BasicPerformanceMetrics.STD_DEV_JITTER": 0.39218231587848257, - "CPUProfilingMetrics.MAX_CPU_UTIL": 16.941666666666666, - "CPUProfilingMetrics.MIN_CPU_UTIL": 14.941666666666665, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 16.181666666666665, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 0.7085862293641089, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 16.708333333333332 + "BasicPerformanceMetrics.MEAN_FRAME_RATE": 60.21149581907896, + "BasicPerformanceMetrics.MAX_JITTER": 3.14453125, + "BasicPerformanceMetrics.MIN_JITTER": 0.00048828125, + "BasicPerformanceMetrics.MEAN_JITTER": 0.41803429110738255, + "BasicPerformanceMetrics.STD_DEV_JITTER": 0.6496207604451233, + "ResourceMetrics.BASELINE_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MAX_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MIN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.BASELINE_OVERALL_CPU_UTILIZATION": 11.083333333333334, + "ResourceMetrics.MEAN_OVERALL_CPU_UTILIZATION": 12.897222222222222, + "ResourceMetrics.STDDEV_OVERALL_CPU_UTILIZATION": 8.347372895663597, + "ResourceMetrics.MAX_OVERALL_CPU_UTILIZATION": 36.0, + "ResourceMetrics.MIN_OVERALL_CPU_UTILIZATION": 0.0 }, "30.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4966.705078125, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 30.200768753097066, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 30.20110871101432, + "BasicPerformanceMetrics.RECEIVED_DURATION": 4965.243408203125, + "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 30.20097361765046, + "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 7.94970703125, + "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 6.4658203125, "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, "BasicPerformanceMetrics.NUM_FRAMES_SENT": 150, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 5.60986328125, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 5.553955078125, - "BasicPerformanceMetrics.MAX_JITTER": 0.810546875, - "BasicPerformanceMetrics.MIN_JITTER": 0.00048828125, - "BasicPerformanceMetrics.MEAN_JITTER": 0.20741395692567569, - "BasicPerformanceMetrics.STD_DEV_JITTER": 0.17099683875789257, - "CPUProfilingMetrics.MAX_CPU_UTIL": 10.4, - "CPUProfilingMetrics.MIN_CPU_UTIL": 8.683333333333334, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 9.2, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 0.6148170459575761, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 8.983333333333333 + "BasicPerformanceMetrics.MEAN_FRAME_RATE": 30.209999322930194, + "BasicPerformanceMetrics.MAX_JITTER": 2.89697265625, + "BasicPerformanceMetrics.MIN_JITTER": 0.00146484375, + "BasicPerformanceMetrics.MEAN_JITTER": 0.3778010187922297, + "BasicPerformanceMetrics.STD_DEV_JITTER": 0.6230465663115995, + "ResourceMetrics.BASELINE_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MAX_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MIN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.BASELINE_OVERALL_CPU_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_OVERALL_CPU_UTILIZATION": 8.223958333333332, + "ResourceMetrics.STDDEV_OVERALL_CPU_UTILIZATION": 8.567716680478572, + "ResourceMetrics.MAX_OVERALL_CPU_UTILIZATION": 30.416666666666668, + "ResourceMetrics.MIN_OVERALL_CPU_UTILIZATION": 0.0 }, "custom": { "data_resolution": "HD (1280,720)" }, "metadata": { "BenchmarkMetadata.NAME": "image_proc::RectifyNode Benchmark", - "BenchmarkMetadata.TEST_FILE_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/scripts/image_proc_rectify_node.py", - "BenchmarkMetadata.TEST_DATETIME": "2023-05-25T06:52:11Z", - "BenchmarkMetadata.DEVICE_HOSTNAME": "ros-ckq37ygg6jfedlqrowtkf66zre", + "BenchmarkMetadata.TEST_FILE_PATH": "/opt/ros/humble/share/ros2_benchmark/scripts/image_proc_rectify_node.py", + "BenchmarkMetadata.TEST_DATETIME": "2024-12-06T14:29:34Z", + "BenchmarkMetadata.DEVICE_HOSTNAME": "5af3bd91161f4c46-1797a96b29d0468a", "BenchmarkMetadata.DEVICE_ARCH": "aarch64", - "BenchmarkMetadata.DEVICE_OS": "Linux 5.10.104-tegra #1 SMP PREEMPT Sun Mar 19 07:55:28 PDT 2023", - "BenchmarkMetadata.IDLE_CPU_UTIL": 1.0833333333333333, + "BenchmarkMetadata.DEVICE_OS": "Linux 5.15.148-tegra #1 SMP PREEMPT Thu Sep 12 21:01:54 PDT 2024", + "BenchmarkMetadata.IDLE_CPU_UTIL": 0.7824074074074076, + "BenchmarkMetadata.IDLE_GPU_UTIL": 0.0, "BenchmarkMetadata.BENCHMARK_MODE": 1, - "BenchmarkMetadata.PEAK_THROUGHPUT_PREDICTION": 185.078125, - "BenchmarkMetadata.INPUT_DATA_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets/datasets/r2b_dataset/r2b_storage", + "BenchmarkMetadata.PEAK_THROUGHPUT_PREDICTION": 160.8984375, + "BenchmarkMetadata.INPUT_DATA_PATH": "/workspaces/isaac_ros-dev/src/ros2_benchmark/assets/datasets/r2b_dataset/r2b_storage", "BenchmarkMetadata.INPUT_DATA_SIZE": 3087908864, "BenchmarkMetadata.INPUT_DATA_HASH": "5e8f11201fe10dbac7307a8628553e94", - "BenchmarkMetadata.CONFIG": "ros2_benchmark_config:\n additional_fixed_publisher_rate_tests:\n - 10.0\n - 30.0\n - 60.0\n assets_root: /workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets\n benchmark_duration: 5.0\n benchmark_mode: BenchmarkMode.LOOPING\n benchmark_name: image_proc::RectifyNode Benchmark\n benchmark_namespace: r2b\n binary_search_acceptable_frame_loss_fraction: 0.05\n binary_search_acceptable_frame_rate_drop: 10.0\n binary_search_duration_fraction: 1.0\n binary_search_terminal_interval_width: 10.0\n cpu_profiling_interval_sec: 1.0\n custom_report_info:\n data_resolution: Resolution(1280, 720, HD)\n default_config_version: 0.30.0\n default_service_future_timeout_sec: 15.0\n enable_cpu_profiler: true\n enable_trial_buffer_preparation: false\n enforce_publisher_rate: false\n input_data_end_time: -1.0\n input_data_path: datasets/r2b_dataset/r2b_storage\n input_data_start_time: -1.0\n linear_scan_acceptable_frame_loss_fraction: 0.05\n linear_scan_acceptable_frame_rate_drop: 5.0\n linear_scan_duration_fraction: 1.0\n linear_scan_step_size: 5.0\n load_data_in_real_time: false\n log_file_name: kpi.json\n log_folder: /osmo/data/output\n monitor_info_list:\n - calculators:\n - class_name: BasicPerformanceCalculator\n config:\n report_prefix: ''\n module_name: ros2_benchmark.basic_performance_calculator\n service_name: start_monitoring0\n play_messages_service_future_timeout_sec: 15.0\n playback_message_buffer_size: 100\n publish_tf_messages_in_set_data: false\n publish_tf_static_messages_in_set_data: false\n publisher_lower_frequency: 10.0\n publisher_upper_frequency: 2500.0\n record_data_timeline: false\n revise_timestamps_as_message_ids: false\n set_data_service_future_timeout_sec: 60.0\n setup_service_client_timeout_sec: 20.0\n start_monitoring_service_timeout_sec: 10\n start_recording_service_future_timeout_sec: 35.0\n start_recording_service_timeout_sec: 30\n test_iterations: 5\n" + "BenchmarkMetadata.CONFIG": "ros2_benchmark_config:\n additional_fixed_publisher_rate_tests:\n - 10.0\n - 30.0\n - 60.0\n assets_root: /workspaces/isaac_ros-dev/src/ros2_benchmark/assets\n benchmark_duration: 5.0\n benchmark_mode: BenchmarkMode.LOOPING\n benchmark_name: image_proc::RectifyNode Benchmark\n benchmark_namespace: r2b\n binary_search_acceptable_frame_loss_fraction: 0.05\n binary_search_acceptable_frame_rate_drop: 10.0\n binary_search_duration_fraction: 1.0\n binary_search_terminal_interval_width: 10.0\n collect_start_timestamps_from_monitors: false\n custom_report_info:\n data_resolution: Resolution(1280, 720, HD)\n default_config_version: 0.30.0\n default_service_future_timeout_sec: 15.0\n enable_resource_profiler: true\n enable_trial_buffer_preparation: false\n enforce_publisher_rate: false\n export_monitor_raw_data: false\n input_data_end_time: -1.0\n input_data_path: datasets/r2b_dataset/r2b_storage\n input_data_start_time: -1.0\n linear_scan_acceptable_frame_loss_fraction: 0.05\n linear_scan_acceptable_frame_rate_drop: 5.0\n linear_scan_duration_fraction: 1.0\n linear_scan_step_size: 5.0\n load_data_in_real_time: false\n log_file_name: kpi.json\n log_folder: /osmo/data/output\n monitor_info_list:\n - calculators:\n - class_name: BasicPerformanceCalculator\n config:\n report_prefix: ''\n module_name: ros2_benchmark.basic_performance_calculator\n service_name: monitor_node0\n play_messages_service_future_timeout_sec: 15.0\n playback_message_buffer_size: 100\n post_trial_run_wait_time_sec: 0.0\n pre_stop_monitoring_wait_time_sec: 5.0\n pre_trial_run_wait_time_sec: 0.0\n publish_tf_messages_in_set_data: false\n publish_tf_static_messages_in_set_data: false\n publisher_lower_frequency: 10.0\n publisher_upper_frequency: 2500.0\n record_data_timeline: false\n resource_profiling_interval_sec: 0.01\n revise_timestamps_as_message_ids: false\n ros1_ws: /workspaces/isaac_ros-dev/isaac_ros_1-dev\n set_data_service_future_timeout_sec: 60.0\n setup_service_client_timeout_sec: 20.0\n start_recording_service_future_timeout_sec: 35.0\n start_recording_service_timeout_sec: 30\n test_iterations: 5\n" } } \ No newline at end of file diff --git a/results/image_proc_rectify_node-agx_xavier.json b/results/image_proc_rectify_node-agx_xavier.json deleted file mode 100644 index 57fd332..0000000 --- a/results/image_proc_rectify_node-agx_xavier.json +++ /dev/null @@ -1,90 +0,0 @@ -{ - "BasicPerformanceMetrics.RECEIVED_DURATION": 5015.636637369792, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 194.9991759197413, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 184.86449853243937, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 46.666666666666664, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 974.0, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 6.776692708333333, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 28.199869791666668, - "BasicPerformanceMetrics.MAX_JITTER": 8.566162109375, - "BasicPerformanceMetrics.MIN_JITTER": 0.0, - "BasicPerformanceMetrics.MEAN_JITTER": 0.6951037650500541, - "BasicPerformanceMetrics.STD_DEV_JITTER": 0.7290898848478206, - "CPUProfilingMetrics.MAX_CPU_UTIL": 59.3, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.5, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 28.9225, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 27.370256723501587, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 58.19583333333333, - "10.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4899.41162109375, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 10.203883354865454, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 10.205307058654105, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 50, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 6.560546875, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 5.876953125, - "BasicPerformanceMetrics.MAX_JITTER": 2.36181640625, - "BasicPerformanceMetrics.MIN_JITTER": 0.012451171875, - "BasicPerformanceMetrics.MEAN_JITTER": 0.5944620768229166, - "BasicPerformanceMetrics.STD_DEV_JITTER": 0.5182798626797805, - "CPUProfilingMetrics.MAX_CPU_UTIL": 3.675, - "CPUProfilingMetrics.MIN_CPU_UTIL": 2.25, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 2.7475, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 0.5240706059301551, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 2.625 - }, - "60.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4981.742431640625, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 60.1988118671023, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 60.21989376540323, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 300, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 7.428466796875, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 5.683837890625, - "BasicPerformanceMetrics.MAX_JITTER": 3.39599609375, - "BasicPerformanceMetrics.MIN_JITTER": 0.00390625, - "BasicPerformanceMetrics.MEAN_JITTER": 0.603515625, - "BasicPerformanceMetrics.STD_DEV_JITTER": 0.5869481683079673, - "CPUProfilingMetrics.MAX_CPU_UTIL": 21.737499999999997, - "CPUProfilingMetrics.MIN_CPU_UTIL": 17.987499999999997, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 19.557499999999997, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 1.2258823352997623, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 19.65 - }, - "30.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4964.74462890625, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 30.200611395254978, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 30.213034347558274, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 150, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 7.12841796875, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 5.086181640625, - "BasicPerformanceMetrics.MAX_JITTER": 4.638427734375, - "BasicPerformanceMetrics.MIN_JITTER": 0.00244140625, - "BasicPerformanceMetrics.MEAN_JITTER": 0.6715516786317568, - "BasicPerformanceMetrics.STD_DEV_JITTER": 0.7759113412194872, - "CPUProfilingMetrics.MAX_CPU_UTIL": 10.625, - "CPUProfilingMetrics.MIN_CPU_UTIL": 8.85, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 9.41, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 0.6493939482317339, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 10.625 - }, - "custom": { - "data_resolution": "HD (1280,720)" - }, - "metadata": { - "BenchmarkMetadata.NAME": "image_proc::RectifyNode Benchmark", - "BenchmarkMetadata.TEST_FILE_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/scripts/image_proc_rectify_node.py", - "BenchmarkMetadata.TEST_DATETIME": "2023-05-25T07:10:50Z", - "BenchmarkMetadata.DEVICE_HOSTNAME": "ros-puzsttk4bzdgvdfvmp6lw63wcy", - "BenchmarkMetadata.DEVICE_ARCH": "aarch64", - "BenchmarkMetadata.DEVICE_OS": "Linux 5.10.104-tegra #1 SMP PREEMPT Sun Mar 19 07:55:28 PDT 2023", - "BenchmarkMetadata.IDLE_CPU_UTIL": 2.25, - "BenchmarkMetadata.BENCHMARK_MODE": 1, - "BenchmarkMetadata.PEAK_THROUGHPUT_PREDICTION": 194.8046875, - "BenchmarkMetadata.INPUT_DATA_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets/datasets/r2b_dataset/r2b_storage", - "BenchmarkMetadata.INPUT_DATA_SIZE": 3087908864, - "BenchmarkMetadata.INPUT_DATA_HASH": "5e8f11201fe10dbac7307a8628553e94", - "BenchmarkMetadata.CONFIG": "ros2_benchmark_config:\n additional_fixed_publisher_rate_tests:\n - 10.0\n - 30.0\n - 60.0\n assets_root: /workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets\n benchmark_duration: 5.0\n benchmark_mode: BenchmarkMode.LOOPING\n benchmark_name: image_proc::RectifyNode Benchmark\n benchmark_namespace: r2b\n binary_search_acceptable_frame_loss_fraction: 0.05\n binary_search_acceptable_frame_rate_drop: 10.0\n binary_search_duration_fraction: 1.0\n binary_search_terminal_interval_width: 10.0\n cpu_profiling_interval_sec: 1.0\n custom_report_info:\n data_resolution: Resolution(1280, 720, HD)\n default_config_version: 0.30.0\n default_service_future_timeout_sec: 15.0\n enable_cpu_profiler: true\n enable_trial_buffer_preparation: false\n enforce_publisher_rate: false\n input_data_end_time: -1.0\n input_data_path: datasets/r2b_dataset/r2b_storage\n input_data_start_time: -1.0\n linear_scan_acceptable_frame_loss_fraction: 0.05\n linear_scan_acceptable_frame_rate_drop: 5.0\n linear_scan_duration_fraction: 1.0\n linear_scan_step_size: 5.0\n load_data_in_real_time: false\n log_file_name: kpi.json\n log_folder: /osmo/data/output\n monitor_info_list:\n - calculators:\n - class_name: BasicPerformanceCalculator\n config:\n report_prefix: ''\n module_name: ros2_benchmark.basic_performance_calculator\n service_name: start_monitoring0\n play_messages_service_future_timeout_sec: 15.0\n playback_message_buffer_size: 100\n publish_tf_messages_in_set_data: false\n publish_tf_static_messages_in_set_data: false\n publisher_lower_frequency: 10.0\n publisher_upper_frequency: 2500.0\n record_data_timeline: false\n revise_timestamps_as_message_ids: false\n set_data_service_future_timeout_sec: 60.0\n setup_service_client_timeout_sec: 20.0\n start_monitoring_service_timeout_sec: 10\n start_recording_service_future_timeout_sec: 35.0\n start_recording_service_timeout_sec: 30\n test_iterations: 5\n" - } -} \ No newline at end of file diff --git a/results/image_proc_rectify_node-nuc_3060ti.json b/results/image_proc_rectify_node-nuc_3060ti.json deleted file mode 100644 index 6d39d93..0000000 --- a/results/image_proc_rectify_node-nuc_3060ti.json +++ /dev/null @@ -1,90 +0,0 @@ -{ - "BasicPerformanceMetrics.RECEIVED_DURATION": 4999.043131510417, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 574.3232250747142, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 566.7736905233289, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 37.0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 2870.0, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 2.0361328125, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 4.056640625, - "BasicPerformanceMetrics.MAX_JITTER": 1.4091796875, - "BasicPerformanceMetrics.MIN_JITTER": 0.0, - "BasicPerformanceMetrics.MEAN_JITTER": 0.11035331073218761, - "BasicPerformanceMetrics.STD_DEV_JITTER": 0.10469953711729131, - "CPUProfilingMetrics.MAX_CPU_UTIL": 34.275, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.1875, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 14.973674242424243, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 15.92230726603812, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 32.458333333333336, - "10.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4904.526611328125, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 10.202959685306448, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 10.194663820258121, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 50, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 1.865966796875, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 5.853759765625, - "BasicPerformanceMetrics.MAX_JITTER": 7.041748046875, - "BasicPerformanceMetrics.MIN_JITTER": 0.024169921875, - "BasicPerformanceMetrics.MEAN_JITTER": 0.8780619303385416, - "BasicPerformanceMetrics.STD_DEV_JITTER": 1.323383867961263, - "CPUProfilingMetrics.MAX_CPU_UTIL": 1.4375, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.625, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 0.9625, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 0.2698379143115363, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 0.9375 - }, - "60.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4987.487060546875, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 60.19563581640332, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 60.150531992980284, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 300, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 1.779541015625, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 5.5166015625, - "BasicPerformanceMetrics.MAX_JITTER": 7.28759765625, - "BasicPerformanceMetrics.MIN_JITTER": 0.000732421875, - "BasicPerformanceMetrics.MEAN_JITTER": 0.7196339856858222, - "BasicPerformanceMetrics.STD_DEV_JITTER": 1.2700523119443694, - "CPUProfilingMetrics.MAX_CPU_UTIL": 11.75625, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.3125, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 8.747916666666667, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 3.824653896832438, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 10.05625 - }, - "30.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4970.625732421875, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 30.198760336956042, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 30.177287141455, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 150, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 1.813232421875, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 5.34765625, - "BasicPerformanceMetrics.MAX_JITTER": 7.57958984375, - "BasicPerformanceMetrics.MIN_JITTER": 0.001220703125, - "BasicPerformanceMetrics.MEAN_JITTER": 0.9473711993243243, - "BasicPerformanceMetrics.STD_DEV_JITTER": 1.5117365079021419, - "CPUProfilingMetrics.MAX_CPU_UTIL": 6.25, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.375, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 4.663541666666667, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 1.9521536473462386, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 6.25 - }, - "custom": { - "data_resolution": "HD (1280,720)" - }, - "metadata": { - "BenchmarkMetadata.NAME": "image_proc::RectifyNode Benchmark", - "BenchmarkMetadata.TEST_FILE_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/scripts/image_proc_rectify_node.py", - "BenchmarkMetadata.TEST_DATETIME": "2023-05-25T06:31:10Z", - "BenchmarkMetadata.DEVICE_HOSTNAME": "ros-dswzkp577ndwdfair4uygxmtcm", - "BenchmarkMetadata.DEVICE_ARCH": "x86_64", - "BenchmarkMetadata.DEVICE_OS": "Linux 5.15.0-50-generic #56~20.04.1-Ubuntu SMP Tue Sep 27 15:51:29 UTC 2022", - "BenchmarkMetadata.IDLE_CPU_UTIL": 0.6875, - "BenchmarkMetadata.BENCHMARK_MODE": 1, - "BenchmarkMetadata.PEAK_THROUGHPUT_PREDICTION": 574.140625, - "BenchmarkMetadata.INPUT_DATA_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets/datasets/r2b_dataset/r2b_storage", - "BenchmarkMetadata.INPUT_DATA_SIZE": 3087908864, - "BenchmarkMetadata.INPUT_DATA_HASH": "5e8f11201fe10dbac7307a8628553e94", - "BenchmarkMetadata.CONFIG": "ros2_benchmark_config:\n additional_fixed_publisher_rate_tests:\n - 10.0\n - 30.0\n - 60.0\n assets_root: /workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets\n benchmark_duration: 5.0\n benchmark_mode: BenchmarkMode.LOOPING\n benchmark_name: image_proc::RectifyNode Benchmark\n benchmark_namespace: r2b\n binary_search_acceptable_frame_loss_fraction: 0.05\n binary_search_acceptable_frame_rate_drop: 10.0\n binary_search_duration_fraction: 1.0\n binary_search_terminal_interval_width: 10.0\n cpu_profiling_interval_sec: 1.0\n custom_report_info:\n data_resolution: Resolution(1280, 720, HD)\n default_config_version: 0.30.0\n default_service_future_timeout_sec: 15.0\n enable_cpu_profiler: true\n enable_trial_buffer_preparation: false\n enforce_publisher_rate: false\n input_data_end_time: -1.0\n input_data_path: datasets/r2b_dataset/r2b_storage\n input_data_start_time: -1.0\n linear_scan_acceptable_frame_loss_fraction: 0.05\n linear_scan_acceptable_frame_rate_drop: 5.0\n linear_scan_duration_fraction: 1.0\n linear_scan_step_size: 5.0\n load_data_in_real_time: false\n log_file_name: kpi.json\n log_folder: /osmo/data/output\n monitor_info_list:\n - calculators:\n - class_name: BasicPerformanceCalculator\n config:\n report_prefix: ''\n module_name: ros2_benchmark.basic_performance_calculator\n service_name: start_monitoring0\n play_messages_service_future_timeout_sec: 15.0\n playback_message_buffer_size: 100\n publish_tf_messages_in_set_data: false\n publish_tf_static_messages_in_set_data: false\n publisher_lower_frequency: 10.0\n publisher_upper_frequency: 2500.0\n record_data_timeline: false\n revise_timestamps_as_message_ids: false\n set_data_service_future_timeout_sec: 60.0\n setup_service_client_timeout_sec: 20.0\n start_monitoring_service_timeout_sec: 10\n start_recording_service_future_timeout_sec: 35.0\n start_recording_service_timeout_sec: 30\n test_iterations: 5\n" - } -} \ No newline at end of file diff --git a/results/image_proc_rectify_node-nuc_4060ti.json b/results/image_proc_rectify_node-nuc_4060ti.json deleted file mode 100644 index 39381cf..0000000 --- a/results/image_proc_rectify_node-nuc_4060ti.json +++ /dev/null @@ -1,90 +0,0 @@ -{ - "BasicPerformanceMetrics.RECEIVED_DURATION": 4999.12158203125, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 550.1353013951385, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 539.2749240915474, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 53.333333333333336, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 2749.0, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 2.3102213541666665, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 4.431315104166667, - "BasicPerformanceMetrics.MAX_JITTER": 1.4423828125, - "BasicPerformanceMetrics.MIN_JITTER": 0.0, - "BasicPerformanceMetrics.MEAN_JITTER": 0.12632677171114232, - "BasicPerformanceMetrics.STD_DEV_JITTER": 0.12809497141610007, - "CPUProfilingMetrics.MAX_CPU_UTIL": 32.05625, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.125, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 14.207386363636362, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 15.077827026250793, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 29.85625, - "10.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4903.73486328125, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 10.203250951049604, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 10.196309832000859, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 50, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 1.90087890625, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 5.23681640625, - "BasicPerformanceMetrics.MAX_JITTER": 3.57421875, - "BasicPerformanceMetrics.MIN_JITTER": 0.004638671875, - "BasicPerformanceMetrics.MEAN_JITTER": 0.362152099609375, - "BasicPerformanceMetrics.STD_DEV_JITTER": 0.5348469372245831, - "CPUProfilingMetrics.MAX_CPU_UTIL": 1.1875, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.625, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 0.8324999999999999, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 0.19674698218778353, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 0.625 - }, - "60.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4986.944091796875, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 60.19614596722808, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 60.15708106563216, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 300, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 2.05712890625, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 5.29345703125, - "BasicPerformanceMetrics.MAX_JITTER": 6.127685546875, - "BasicPerformanceMetrics.MIN_JITTER": 0.0009765625, - "BasicPerformanceMetrics.MEAN_JITTER": 0.6985789049391778, - "BasicPerformanceMetrics.STD_DEV_JITTER": 1.0545648249978745, - "CPUProfilingMetrics.MAX_CPU_UTIL": 8.85625, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.1875, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 6.8500000000000005, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 3.0202161725887544, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 8.225 - }, - "30.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4970.196044921875, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 30.198784086066535, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 30.17989605324669, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 150, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 1.8994140625, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 5.008056640625, - "BasicPerformanceMetrics.MAX_JITTER": 6.71630859375, - "BasicPerformanceMetrics.MIN_JITTER": 0.00048828125, - "BasicPerformanceMetrics.MEAN_JITTER": 1.0057884422508445, - "BasicPerformanceMetrics.STD_DEV_JITTER": 1.4835791029770782, - "CPUProfilingMetrics.MAX_CPU_UTIL": 5.625, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.375, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 4.0875, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 1.7207942388618112, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 5.16875 - }, - "custom": { - "data_resolution": "HD (1280,720)" - }, - "metadata": { - "BenchmarkMetadata.NAME": "image_proc::RectifyNode Benchmark", - "BenchmarkMetadata.TEST_FILE_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/scripts/image_proc_rectify_node.py", - "BenchmarkMetadata.TEST_DATETIME": "2023-05-25T07:55:46Z", - "BenchmarkMetadata.DEVICE_HOSTNAME": "ros-3lzudg52gvdffpzzybb6fackvy", - "BenchmarkMetadata.DEVICE_ARCH": "x86_64", - "BenchmarkMetadata.DEVICE_OS": "Linux 5.15.0-72-generic #79~20.04.1-Ubuntu SMP Thu Apr 20 22:12:07 UTC 2023", - "BenchmarkMetadata.IDLE_CPU_UTIL": 0.6875, - "BenchmarkMetadata.BENCHMARK_MODE": 1, - "BenchmarkMetadata.PEAK_THROUGHPUT_PREDICTION": 549.9609375, - "BenchmarkMetadata.INPUT_DATA_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets/datasets/r2b_dataset/r2b_storage", - "BenchmarkMetadata.INPUT_DATA_SIZE": 3087908864, - "BenchmarkMetadata.INPUT_DATA_HASH": "5e8f11201fe10dbac7307a8628553e94", - "BenchmarkMetadata.CONFIG": "ros2_benchmark_config:\n additional_fixed_publisher_rate_tests:\n - 10.0\n - 30.0\n - 60.0\n assets_root: /workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets\n benchmark_duration: 5.0\n benchmark_mode: BenchmarkMode.LOOPING\n benchmark_name: image_proc::RectifyNode Benchmark\n benchmark_namespace: r2b\n binary_search_acceptable_frame_loss_fraction: 0.05\n binary_search_acceptable_frame_rate_drop: 10.0\n binary_search_duration_fraction: 1.0\n binary_search_terminal_interval_width: 10.0\n cpu_profiling_interval_sec: 1.0\n custom_report_info:\n data_resolution: Resolution(1280, 720, HD)\n default_config_version: 0.30.0\n default_service_future_timeout_sec: 15.0\n enable_cpu_profiler: true\n enable_trial_buffer_preparation: false\n enforce_publisher_rate: false\n input_data_end_time: -1.0\n input_data_path: datasets/r2b_dataset/r2b_storage\n input_data_start_time: -1.0\n linear_scan_acceptable_frame_loss_fraction: 0.05\n linear_scan_acceptable_frame_rate_drop: 5.0\n linear_scan_duration_fraction: 1.0\n linear_scan_step_size: 5.0\n load_data_in_real_time: false\n log_file_name: kpi.json\n log_folder: /osmo/data/output\n monitor_info_list:\n - calculators:\n - class_name: BasicPerformanceCalculator\n config:\n report_prefix: ''\n module_name: ros2_benchmark.basic_performance_calculator\n service_name: start_monitoring0\n play_messages_service_future_timeout_sec: 15.0\n playback_message_buffer_size: 100\n publish_tf_messages_in_set_data: false\n publish_tf_static_messages_in_set_data: false\n publisher_lower_frequency: 10.0\n publisher_upper_frequency: 2500.0\n record_data_timeline: false\n revise_timestamps_as_message_ids: false\n set_data_service_future_timeout_sec: 60.0\n setup_service_client_timeout_sec: 20.0\n start_monitoring_service_timeout_sec: 10\n start_recording_service_future_timeout_sec: 35.0\n start_recording_service_timeout_sec: 30\n test_iterations: 5\n" - } -} \ No newline at end of file diff --git a/results/image_proc_rectify_node-orin_nx.json b/results/image_proc_rectify_node-orin_nx.json index 858085f..49a5553 100644 --- a/results/image_proc_rectify_node-orin_nx.json +++ b/results/image_proc_rectify_node-orin_nx.json @@ -1,90 +1,111 @@ { - "BasicPerformanceMetrics.RECEIVED_DURATION": 5003.564208984375, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 165.82207040719558, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 161.20544657847768, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 21.666666666666668, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 828.0, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 7.798665364583333, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 17.166259765625, - "BasicPerformanceMetrics.MAX_JITTER": 6.04296875, - "BasicPerformanceMetrics.MIN_JITTER": 0.0, - "BasicPerformanceMetrics.MEAN_JITTER": 0.7994316527562112, - "BasicPerformanceMetrics.STD_DEV_JITTER": 0.589160057328505, - "CPUProfilingMetrics.MAX_CPU_UTIL": 68.8375, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.25, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 34.85333333333333, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 32.80144798893976, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 66.60833333333333, + "BasicPerformanceMetrics.RECEIVED_DURATION": 4990.028483072917, + "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 136.6445345759186, + "BasicPerformanceMetrics.MEAN_FRAME_RATE": 134.08657241676715, + "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 13.0, + "BasicPerformanceMetrics.NUM_FRAMES_SENT": 682.0, + "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 8.329345703125, + "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 7.447916666666667, + "BasicPerformanceMetrics.MAX_JITTER": 10.677490234375, + "BasicPerformanceMetrics.MIN_JITTER": 0.000732421875, + "BasicPerformanceMetrics.MEAN_JITTER": 0.8227807066494361, + "BasicPerformanceMetrics.STD_DEV_JITTER": 0.6962732944951106, + "ResourceMetrics.BASELINE_OVERALL_CPU_UTILIZATION": 56.916666666666664, + "ResourceMetrics.MAX_OVERALL_CPU_UTILIZATION": 93.75, + "ResourceMetrics.MIN_OVERALL_CPU_UTILIZATION": 18.75, + "ResourceMetrics.MEAN_OVERALL_CPU_UTILIZATION": 68.115234375, + "ResourceMetrics.STDDEV_OVERALL_CPU_UTILIZATION": 12.414341341489513, + "ResourceMetrics.BASELINE_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MAX_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MIN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_DEVICE_UTILIZATION": 0.0, "10.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4901.598876953125, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 10.203804554294779, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 10.200753112437551, + "BasicPerformanceMetrics.RECEIVED_DURATION": 4900.342529296875, + "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 10.203865052688911, + "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 7.3564453125, + "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 7.594970703125, "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, "BasicPerformanceMetrics.NUM_FRAMES_SENT": 50, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 6.695556640625, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 8.161376953125, - "BasicPerformanceMetrics.MAX_JITTER": 3.7392578125, - "BasicPerformanceMetrics.MIN_JITTER": 0.005859375, - "BasicPerformanceMetrics.MEAN_JITTER": 0.8769175211588541, - "BasicPerformanceMetrics.STD_DEV_JITTER": 0.937031300246649, - "CPUProfilingMetrics.MAX_CPU_UTIL": 6.4625, - "CPUProfilingMetrics.MIN_CPU_UTIL": 4.5, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 5.202500000000001, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 0.7136350607978845, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 6.4625 + "BasicPerformanceMetrics.MEAN_FRAME_RATE": 10.203368377021237, + "BasicPerformanceMetrics.MAX_JITTER": 1.7890625, + "BasicPerformanceMetrics.MIN_JITTER": 0.010498046875, + "BasicPerformanceMetrics.MEAN_JITTER": 0.6587626139322916, + "BasicPerformanceMetrics.STD_DEV_JITTER": 0.4260093910640211, + "ResourceMetrics.BASELINE_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MAX_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MIN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.BASELINE_OVERALL_CPU_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_OVERALL_CPU_UTILIZATION": 4.329656862745098, + "ResourceMetrics.STDDEV_OVERALL_CPU_UTILIZATION": 8.318745612093654, + "ResourceMetrics.MAX_OVERALL_CPU_UTILIZATION": 43.75, + "ResourceMetrics.MIN_OVERALL_CPU_UTILIZATION": 0.0 }, "60.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4981.9091796875, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 60.19906549377123, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 60.21787816268824, + "BasicPerformanceMetrics.RECEIVED_DURATION": 4983.015625, + "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 60.200186195335, + "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 7.49462890625, + "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 7.136962890625, "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, "BasicPerformanceMetrics.NUM_FRAMES_SENT": 300, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 8.27978515625, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 6.722900390625, - "BasicPerformanceMetrics.MAX_JITTER": 6.090087890625, - "BasicPerformanceMetrics.MIN_JITTER": 0.00634765625, - "BasicPerformanceMetrics.MEAN_JITTER": 0.5115081991925335, - "BasicPerformanceMetrics.STD_DEV_JITTER": 0.7261506276608013, - "CPUProfilingMetrics.MAX_CPU_UTIL": 24.237499999999997, - "CPUProfilingMetrics.MIN_CPU_UTIL": 1.625, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 20.297916666666666, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 8.353942484523234, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 24.237499999999997 + "BasicPerformanceMetrics.MEAN_FRAME_RATE": 60.204507185345214, + "BasicPerformanceMetrics.MAX_JITTER": 6.742919921875, + "BasicPerformanceMetrics.MIN_JITTER": 0.001953125, + "BasicPerformanceMetrics.MEAN_JITTER": 0.726318359375, + "BasicPerformanceMetrics.STD_DEV_JITTER": 0.7252780693568461, + "ResourceMetrics.BASELINE_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MAX_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MIN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.BASELINE_OVERALL_CPU_UTILIZATION": 12.5, + "ResourceMetrics.MEAN_OVERALL_CPU_UTILIZATION": 25.4140625, + "ResourceMetrics.STDDEV_OVERALL_CPU_UTILIZATION": 10.792028315509574, + "ResourceMetrics.MAX_OVERALL_CPU_UTILIZATION": 59.125, + "ResourceMetrics.MIN_OVERALL_CPU_UTILIZATION": 0.0 }, "30.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4963.4111328125, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 30.200540139413043, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 30.22115153999, + "BasicPerformanceMetrics.RECEIVED_DURATION": 4964.54052734375, + "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 30.200816257673523, + "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 9.740234375, + "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 7.527587890625, "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, "BasicPerformanceMetrics.NUM_FRAMES_SENT": 150, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 9.78369140625, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 6.396240234375, - "BasicPerformanceMetrics.MAX_JITTER": 3.810791015625, - "BasicPerformanceMetrics.MIN_JITTER": 0.004638671875, - "BasicPerformanceMetrics.MEAN_JITTER": 0.36727657833614863, - "BasicPerformanceMetrics.STD_DEV_JITTER": 0.4854342686851023, - "CPUProfilingMetrics.MAX_CPU_UTIL": 13.55, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.625, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 10.729166666666666, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 4.538749062853724, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 12.649999999999999 + "BasicPerformanceMetrics.MEAN_FRAME_RATE": 30.214276462006577, + "BasicPerformanceMetrics.MAX_JITTER": 6.858154296875, + "BasicPerformanceMetrics.MIN_JITTER": 0.0048828125, + "BasicPerformanceMetrics.MEAN_JITTER": 0.7180885108741554, + "BasicPerformanceMetrics.STD_DEV_JITTER": 0.9329792966635243, + "ResourceMetrics.BASELINE_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MAX_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MIN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.BASELINE_OVERALL_CPU_UTILIZATION": 4.125, + "ResourceMetrics.MEAN_OVERALL_CPU_UTILIZATION": 11.826923076923077, + "ResourceMetrics.STDDEV_OVERALL_CPU_UTILIZATION": 12.442000551038067, + "ResourceMetrics.MAX_OVERALL_CPU_UTILIZATION": 45.75, + "ResourceMetrics.MIN_OVERALL_CPU_UTILIZATION": 0.0 }, "custom": { "data_resolution": "HD (1280,720)" }, "metadata": { "BenchmarkMetadata.NAME": "image_proc::RectifyNode Benchmark", - "BenchmarkMetadata.TEST_FILE_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/scripts/image_proc_rectify_node.py", - "BenchmarkMetadata.TEST_DATETIME": "2023-05-25T06:20:33Z", - "BenchmarkMetadata.DEVICE_HOSTNAME": "ros-nwy2tusjinek7hsartfhnjtnwi", + "BenchmarkMetadata.TEST_FILE_PATH": "/opt/ros/humble/share/ros2_benchmark/scripts/image_proc_rectify_node.py", + "BenchmarkMetadata.TEST_DATETIME": "2024-12-06T13:02:18Z", + "BenchmarkMetadata.DEVICE_HOSTNAME": "bde2f93774fe4002-42b4322275a549a2", "BenchmarkMetadata.DEVICE_ARCH": "aarch64", - "BenchmarkMetadata.DEVICE_OS": "Linux 5.10.104-tegra #1 SMP PREEMPT Wed Mar 15 00:17:07 PDT 2023", - "BenchmarkMetadata.IDLE_CPU_UTIL": 0.125, + "BenchmarkMetadata.DEVICE_OS": "Linux 5.15.148-tegra #1 SMP PREEMPT Thu Sep 12 21:01:54 PDT 2024", + "BenchmarkMetadata.IDLE_CPU_UTIL": 0.6111111111111112, + "BenchmarkMetadata.IDLE_GPU_UTIL": 0.0, "BenchmarkMetadata.BENCHMARK_MODE": 1, - "BenchmarkMetadata.PEAK_THROUGHPUT_PREDICTION": 165.625, - "BenchmarkMetadata.INPUT_DATA_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets/datasets/r2b_dataset/r2b_storage", + "BenchmarkMetadata.PEAK_THROUGHPUT_PREDICTION": 136.4453125, + "BenchmarkMetadata.INPUT_DATA_PATH": "/workspaces/isaac_ros-dev/src/ros2_benchmark/assets/datasets/r2b_dataset/r2b_storage", "BenchmarkMetadata.INPUT_DATA_SIZE": 3087908864, "BenchmarkMetadata.INPUT_DATA_HASH": "5e8f11201fe10dbac7307a8628553e94", - "BenchmarkMetadata.CONFIG": "ros2_benchmark_config:\n additional_fixed_publisher_rate_tests:\n - 10.0\n - 30.0\n - 60.0\n assets_root: /workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets\n benchmark_duration: 5.0\n benchmark_mode: BenchmarkMode.LOOPING\n benchmark_name: image_proc::RectifyNode Benchmark\n benchmark_namespace: r2b\n binary_search_acceptable_frame_loss_fraction: 0.05\n binary_search_acceptable_frame_rate_drop: 10.0\n binary_search_duration_fraction: 1.0\n binary_search_terminal_interval_width: 10.0\n cpu_profiling_interval_sec: 1.0\n custom_report_info:\n data_resolution: Resolution(1280, 720, HD)\n default_config_version: 0.30.0\n default_service_future_timeout_sec: 15.0\n enable_cpu_profiler: true\n enable_trial_buffer_preparation: false\n enforce_publisher_rate: false\n input_data_end_time: -1.0\n input_data_path: datasets/r2b_dataset/r2b_storage\n input_data_start_time: -1.0\n linear_scan_acceptable_frame_loss_fraction: 0.05\n linear_scan_acceptable_frame_rate_drop: 5.0\n linear_scan_duration_fraction: 1.0\n linear_scan_step_size: 5.0\n load_data_in_real_time: false\n log_file_name: kpi.json\n log_folder: /osmo/data/output\n monitor_info_list:\n - calculators:\n - class_name: BasicPerformanceCalculator\n config:\n report_prefix: ''\n module_name: ros2_benchmark.basic_performance_calculator\n service_name: start_monitoring0\n play_messages_service_future_timeout_sec: 15.0\n playback_message_buffer_size: 100\n publish_tf_messages_in_set_data: false\n publish_tf_static_messages_in_set_data: false\n publisher_lower_frequency: 10.0\n publisher_upper_frequency: 2500.0\n record_data_timeline: false\n revise_timestamps_as_message_ids: false\n set_data_service_future_timeout_sec: 60.0\n setup_service_client_timeout_sec: 20.0\n start_monitoring_service_timeout_sec: 10\n start_recording_service_future_timeout_sec: 35.0\n start_recording_service_timeout_sec: 30\n test_iterations: 5\n" + "BenchmarkMetadata.CONFIG": "ros2_benchmark_config:\n additional_fixed_publisher_rate_tests:\n - 10.0\n - 30.0\n - 60.0\n assets_root: /workspaces/isaac_ros-dev/src/ros2_benchmark/assets\n benchmark_duration: 5.0\n benchmark_mode: BenchmarkMode.LOOPING\n benchmark_name: image_proc::RectifyNode Benchmark\n benchmark_namespace: r2b\n binary_search_acceptable_frame_loss_fraction: 0.05\n binary_search_acceptable_frame_rate_drop: 10.0\n binary_search_duration_fraction: 1.0\n binary_search_terminal_interval_width: 10.0\n collect_start_timestamps_from_monitors: false\n custom_report_info:\n data_resolution: Resolution(1280, 720, HD)\n default_config_version: 0.30.0\n default_service_future_timeout_sec: 15.0\n enable_resource_profiler: true\n enable_trial_buffer_preparation: false\n enforce_publisher_rate: false\n export_monitor_raw_data: false\n input_data_end_time: -1.0\n input_data_path: datasets/r2b_dataset/r2b_storage\n input_data_start_time: -1.0\n linear_scan_acceptable_frame_loss_fraction: 0.05\n linear_scan_acceptable_frame_rate_drop: 5.0\n linear_scan_duration_fraction: 1.0\n linear_scan_step_size: 5.0\n load_data_in_real_time: false\n log_file_name: kpi.json\n log_folder: /osmo/data/output\n monitor_info_list:\n - calculators:\n - class_name: BasicPerformanceCalculator\n config:\n report_prefix: ''\n module_name: ros2_benchmark.basic_performance_calculator\n service_name: monitor_node0\n play_messages_service_future_timeout_sec: 15.0\n playback_message_buffer_size: 100\n post_trial_run_wait_time_sec: 0.0\n pre_stop_monitoring_wait_time_sec: 5.0\n pre_trial_run_wait_time_sec: 0.0\n publish_tf_messages_in_set_data: false\n publish_tf_static_messages_in_set_data: false\n publisher_lower_frequency: 10.0\n publisher_upper_frequency: 2500.0\n record_data_timeline: false\n resource_profiling_interval_sec: 0.01\n revise_timestamps_as_message_ids: false\n ros1_ws: /workspaces/isaac_ros-dev/isaac_ros_1-dev\n set_data_service_future_timeout_sec: 60.0\n setup_service_client_timeout_sec: 20.0\n start_recording_service_future_timeout_sec: 35.0\n start_recording_service_timeout_sec: 30\n test_iterations: 5\n" } } \ No newline at end of file diff --git a/results/image_proc_rectify_node-x86-4090.json b/results/image_proc_rectify_node-x86-4090.json new file mode 100644 index 0000000..6ee6976 --- /dev/null +++ b/results/image_proc_rectify_node-x86-4090.json @@ -0,0 +1,131 @@ +{ + "BasicPerformanceMetrics.RECEIVED_DURATION": 4998.361083984375, + "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 535.4395273622612, + "BasicPerformanceMetrics.MEAN_FRAME_RATE": 528.1725821325671, + "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 36.333333333333336, + "BasicPerformanceMetrics.NUM_FRAMES_SENT": 2676.0, + "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 2.7013346354166665, + "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 3.2586263020833335, + "BasicPerformanceMetrics.MAX_JITTER": 1.94677734375, + "BasicPerformanceMetrics.MIN_JITTER": 0.0, + "BasicPerformanceMetrics.MEAN_JITTER": 0.3120672052556818, + "BasicPerformanceMetrics.STD_DEV_JITTER": 0.2566642231033513, + "ResourceMetrics.BASELINE_OVERALL_CPU_UTILIZATION": 8.333333333333334, + "ResourceMetrics.MAX_OVERALL_CPU_UTILIZATION": 31.25, + "ResourceMetrics.MIN_OVERALL_CPU_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_OVERALL_CPU_UTILIZATION": 5.204895873497847, + "ResourceMetrics.STDDEV_OVERALL_CPU_UTILIZATION": 6.527751854526573, + "ResourceMetrics.BASELINE_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MAX_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MIN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.BASELINE_DEVICE_MEMORY_UTILIZATION": 1.9400000000000002, + "ResourceMetrics.MAX_DEVICE_MEMORY_UTILIZATION": 1.94, + "ResourceMetrics.MIN_DEVICE_MEMORY_UTILIZATION": 1.94, + "ResourceMetrics.MEAN_DEVICE_MEMORY_UTILIZATION": 1.9400000000000002, + "ResourceMetrics.STDDEV_DEVICE_MEMORY_UTILIZATION": 2.220446049250313e-16, + "10.0fps": { + "BasicPerformanceMetrics.RECEIVED_DURATION": 4900.16357421875, + "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 10.204382622893053, + "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 2.6806640625, + "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 2.98876953125, + "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, + "BasicPerformanceMetrics.NUM_FRAMES_SENT": 50, + "BasicPerformanceMetrics.MEAN_FRAME_RATE": 10.203741006333992, + "BasicPerformanceMetrics.MAX_JITTER": 1.277587890625, + "BasicPerformanceMetrics.MIN_JITTER": 0.009521484375, + "BasicPerformanceMetrics.MEAN_JITTER": 0.4923044840494792, + "BasicPerformanceMetrics.STD_DEV_JITTER": 0.3405950798139701, + "ResourceMetrics.BASELINE_OVERALL_CPU_UTILIZATION": 7.8125, + "ResourceMetrics.MEAN_OVERALL_CPU_UTILIZATION": 0.12853406652360516, + "ResourceMetrics.MAX_OVERALL_CPU_UTILIZATION": 7.8125, + "ResourceMetrics.MIN_OVERALL_CPU_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_OVERALL_CPU_UTILIZATION": 0.5712821273162917, + "ResourceMetrics.BASELINE_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MAX_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MIN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.BASELINE_DEVICE_MEMORY_UTILIZATION": 1.94, + "ResourceMetrics.MEAN_DEVICE_MEMORY_UTILIZATION": 1.9400000000000002, + "ResourceMetrics.STDDEV_DEVICE_MEMORY_UTILIZATION": 2.220446049250313e-16, + "ResourceMetrics.MAX_DEVICE_MEMORY_UTILIZATION": 1.94, + "ResourceMetrics.MIN_DEVICE_MEMORY_UTILIZATION": 1.94 + }, + "60.0fps": { + "BasicPerformanceMetrics.RECEIVED_DURATION": 4984.218994140625, + "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 60.202035444144336, + "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 2.525634765625, + "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 3.5244140625, + "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, + "BasicPerformanceMetrics.NUM_FRAMES_SENT": 300, + "BasicPerformanceMetrics.MEAN_FRAME_RATE": 60.18997165908553, + "BasicPerformanceMetrics.MAX_JITTER": 2.52587890625, + "BasicPerformanceMetrics.MIN_JITTER": 0.0009765625, + "BasicPerformanceMetrics.MEAN_JITTER": 0.5728030620805369, + "BasicPerformanceMetrics.STD_DEV_JITTER": 0.4515256644553695, + "ResourceMetrics.BASELINE_OVERALL_CPU_UTILIZATION": 1.5625, + "ResourceMetrics.MEAN_OVERALL_CPU_UTILIZATION": 0.5398336909871244, + "ResourceMetrics.MAX_OVERALL_CPU_UTILIZATION": 10.9375, + "ResourceMetrics.MIN_OVERALL_CPU_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_OVERALL_CPU_UTILIZATION": 1.492840203828041, + "ResourceMetrics.BASELINE_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MAX_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MIN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.BASELINE_DEVICE_MEMORY_UTILIZATION": 1.94, + "ResourceMetrics.MEAN_DEVICE_MEMORY_UTILIZATION": 1.9400000000000002, + "ResourceMetrics.STDDEV_DEVICE_MEMORY_UTILIZATION": 2.220446049250313e-16, + "ResourceMetrics.MAX_DEVICE_MEMORY_UTILIZATION": 1.94, + "ResourceMetrics.MIN_DEVICE_MEMORY_UTILIZATION": 1.94 + }, + "30.0fps": { + "BasicPerformanceMetrics.RECEIVED_DURATION": 4966.218017578125, + "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 30.201983135751377, + "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 2.87939453125, + "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 2.5361328125, + "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, + "BasicPerformanceMetrics.NUM_FRAMES_SENT": 150, + "BasicPerformanceMetrics.MEAN_FRAME_RATE": 30.20407067693546, + "BasicPerformanceMetrics.MAX_JITTER": 2.12890625, + "BasicPerformanceMetrics.MIN_JITTER": 0.001953125, + "BasicPerformanceMetrics.MEAN_JITTER": 0.5482705605996622, + "BasicPerformanceMetrics.STD_DEV_JITTER": 0.44022052927687755, + "ResourceMetrics.BASELINE_OVERALL_CPU_UTILIZATION": 4.6875, + "ResourceMetrics.MEAN_OVERALL_CPU_UTILIZATION": 0.3688304721030043, + "ResourceMetrics.MAX_OVERALL_CPU_UTILIZATION": 9.375, + "ResourceMetrics.MIN_OVERALL_CPU_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_OVERALL_CPU_UTILIZATION": 1.2495689279087634, + "ResourceMetrics.BASELINE_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MAX_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MIN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.BASELINE_DEVICE_MEMORY_UTILIZATION": 1.94, + "ResourceMetrics.MEAN_DEVICE_MEMORY_UTILIZATION": 1.9400000000000002, + "ResourceMetrics.STDDEV_DEVICE_MEMORY_UTILIZATION": 2.220446049250313e-16, + "ResourceMetrics.MAX_DEVICE_MEMORY_UTILIZATION": 1.94, + "ResourceMetrics.MIN_DEVICE_MEMORY_UTILIZATION": 1.94 + }, + "custom": { + "data_resolution": "HD (1280,720)" + }, + "metadata": { + "BenchmarkMetadata.NAME": "image_proc::RectifyNode Benchmark", + "BenchmarkMetadata.TEST_FILE_PATH": "/opt/ros/humble/share/ros2_benchmark/scripts/image_proc_rectify_node.py", + "BenchmarkMetadata.TEST_DATETIME": "2024-12-06T14:35:07Z", + "BenchmarkMetadata.DEVICE_HOSTNAME": "b509c5ce01b641a6-0046f46d35ea4ffb", + "BenchmarkMetadata.DEVICE_ARCH": "x86_64", + "BenchmarkMetadata.DEVICE_OS": "Linux 6.8.0-48-generic #48~22.04.1-Ubuntu SMP PREEMPT_DYNAMIC Mon Oct 7 11:24:13 UTC 2", + "BenchmarkMetadata.IDLE_CPU_UTIL": 0.125, + "BenchmarkMetadata.IDLE_GPU_UTIL": 0, + "BenchmarkMetadata.BENCHMARK_MODE": 1, + "BenchmarkMetadata.PEAK_THROUGHPUT_PREDICTION": 535.234375, + "BenchmarkMetadata.INPUT_DATA_PATH": "/workspaces/isaac_ros-dev/src/ros2_benchmark/assets/datasets/r2b_dataset/r2b_storage", + "BenchmarkMetadata.INPUT_DATA_SIZE": 3087908864, + "BenchmarkMetadata.INPUT_DATA_HASH": "5e8f11201fe10dbac7307a8628553e94", + "BenchmarkMetadata.CONFIG": "ros2_benchmark_config:\n additional_fixed_publisher_rate_tests:\n - 10.0\n - 30.0\n - 60.0\n assets_root: /workspaces/isaac_ros-dev/src/ros2_benchmark/assets\n benchmark_duration: 5.0\n benchmark_mode: BenchmarkMode.LOOPING\n benchmark_name: image_proc::RectifyNode Benchmark\n benchmark_namespace: r2b\n binary_search_acceptable_frame_loss_fraction: 0.05\n binary_search_acceptable_frame_rate_drop: 10.0\n binary_search_duration_fraction: 1.0\n binary_search_terminal_interval_width: 10.0\n collect_start_timestamps_from_monitors: false\n custom_report_info:\n data_resolution: Resolution(1280, 720, HD)\n default_config_version: 0.30.0\n default_service_future_timeout_sec: 15.0\n enable_resource_profiler: true\n enable_trial_buffer_preparation: false\n enforce_publisher_rate: false\n export_monitor_raw_data: false\n input_data_end_time: -1.0\n input_data_path: datasets/r2b_dataset/r2b_storage\n input_data_start_time: -1.0\n linear_scan_acceptable_frame_loss_fraction: 0.05\n linear_scan_acceptable_frame_rate_drop: 5.0\n linear_scan_duration_fraction: 1.0\n linear_scan_step_size: 5.0\n load_data_in_real_time: false\n log_file_name: kpi.json\n log_folder: /osmo/data/output\n monitor_info_list:\n - calculators:\n - class_name: BasicPerformanceCalculator\n config:\n report_prefix: ''\n module_name: ros2_benchmark.basic_performance_calculator\n service_name: monitor_node0\n play_messages_service_future_timeout_sec: 15.0\n playback_message_buffer_size: 100\n post_trial_run_wait_time_sec: 0.0\n pre_stop_monitoring_wait_time_sec: 5.0\n pre_trial_run_wait_time_sec: 0.0\n publish_tf_messages_in_set_data: false\n publish_tf_static_messages_in_set_data: false\n publisher_lower_frequency: 10.0\n publisher_upper_frequency: 2500.0\n record_data_timeline: false\n resource_profiling_interval_sec: 0.01\n revise_timestamps_as_message_ids: false\n ros1_ws: /workspaces/isaac_ros-dev/isaac_ros_1-dev\n set_data_service_future_timeout_sec: 60.0\n setup_service_client_timeout_sec: 20.0\n start_recording_service_future_timeout_sec: 35.0\n start_recording_service_timeout_sec: 30\n test_iterations: 5\n" + } +} \ No newline at end of file diff --git a/results/image_proc_rectify_node-x86_4090.json b/results/image_proc_rectify_node-x86_4090.json deleted file mode 100644 index 9396090..0000000 --- a/results/image_proc_rectify_node-x86_4090.json +++ /dev/null @@ -1,90 +0,0 @@ -{ - "BasicPerformanceMetrics.RECEIVED_DURATION": 4997.389729817708, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 627.9556064375505, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 623.3974720200036, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 22.333333333333332, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 3138.0, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 2.021240234375, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 2.2628580729166665, - "BasicPerformanceMetrics.MAX_JITTER": 3.060302734375, - "BasicPerformanceMetrics.MIN_JITTER": 0.0, - "BasicPerformanceMetrics.MEAN_JITTER": 0.07386976467072484, - "BasicPerformanceMetrics.STD_DEV_JITTER": 0.1235470236289391, - "CPUProfilingMetrics.MAX_CPU_UTIL": 37.91875, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.1875, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 16.109280303030303, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 17.13742357296611, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 36.10833333333334, - "10.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4905.3203125, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 10.203092354027774, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 10.193014281368603, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 50, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 1.823974609375, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 6.669189453125, - "BasicPerformanceMetrics.MAX_JITTER": 8.4150390625, - "BasicPerformanceMetrics.MIN_JITTER": 0.00830078125, - "BasicPerformanceMetrics.MEAN_JITTER": 1.2941335042317708, - "BasicPerformanceMetrics.STD_DEV_JITTER": 2.0145162459453, - "CPUProfilingMetrics.MAX_CPU_UTIL": 4.1375, - "CPUProfilingMetrics.MIN_CPU_UTIL": 3.5375, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 3.8675000000000006, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 0.22204307014631183, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 3.5375 - }, - "60.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4989.693115234375, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 60.19349504652798, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 60.12393810033114, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 300, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 1.607177734375, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 7.373046875, - "BasicPerformanceMetrics.MAX_JITTER": 13.865966796875, - "BasicPerformanceMetrics.MIN_JITTER": 0.0009765625, - "BasicPerformanceMetrics.MEAN_JITTER": 1.2366492764261745, - "BasicPerformanceMetrics.STD_DEV_JITTER": 2.6013298036833072, - "CPUProfilingMetrics.MAX_CPU_UTIL": 15.09375, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.75, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 11.951041666666667, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 5.052740614460027, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 14.03125 - }, - "30.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4972.449462890625, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 30.197776266032186, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 30.166219107795776, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 150, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 1.846435546875, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 7.042724609375, - "BasicPerformanceMetrics.MAX_JITTER": 12.63525390625, - "BasicPerformanceMetrics.MIN_JITTER": 0.005615234375, - "BasicPerformanceMetrics.MEAN_JITTER": 0.9751553922086149, - "BasicPerformanceMetrics.STD_DEV_JITTER": 2.0733802234320797, - "CPUProfilingMetrics.MAX_CPU_UTIL": 8.5375, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.125, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 6.505208333333333, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 2.899194878958735, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 6.89375 - }, - "custom": { - "data_resolution": "HD (1280,720)" - }, - "metadata": { - "BenchmarkMetadata.NAME": "image_proc::RectifyNode Benchmark", - "BenchmarkMetadata.TEST_FILE_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/scripts/image_proc_rectify_node.py", - "BenchmarkMetadata.TEST_DATETIME": "2023-05-25T07:04:41Z", - "BenchmarkMetadata.DEVICE_HOSTNAME": "ros-5lzdpv32ojdlnkb7kuvmmc7z5y", - "BenchmarkMetadata.DEVICE_ARCH": "x86_64", - "BenchmarkMetadata.DEVICE_OS": "Linux 5.4.0-131-generic #147-Ubuntu SMP Fri Oct 14 17:07:22 UTC 2022", - "BenchmarkMetadata.IDLE_CPU_UTIL": 0.5, - "BenchmarkMetadata.BENCHMARK_MODE": 1, - "BenchmarkMetadata.PEAK_THROUGHPUT_PREDICTION": 627.7734375, - "BenchmarkMetadata.INPUT_DATA_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets/datasets/r2b_dataset/r2b_storage", - "BenchmarkMetadata.INPUT_DATA_SIZE": 3087908864, - "BenchmarkMetadata.INPUT_DATA_HASH": "5e8f11201fe10dbac7307a8628553e94", - "BenchmarkMetadata.CONFIG": "ros2_benchmark_config:\n additional_fixed_publisher_rate_tests:\n - 10.0\n - 30.0\n - 60.0\n assets_root: /workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets\n benchmark_duration: 5.0\n benchmark_mode: BenchmarkMode.LOOPING\n benchmark_name: image_proc::RectifyNode Benchmark\n benchmark_namespace: r2b\n binary_search_acceptable_frame_loss_fraction: 0.05\n binary_search_acceptable_frame_rate_drop: 10.0\n binary_search_duration_fraction: 1.0\n binary_search_terminal_interval_width: 10.0\n cpu_profiling_interval_sec: 1.0\n custom_report_info:\n data_resolution: Resolution(1280, 720, HD)\n default_config_version: 0.30.0\n default_service_future_timeout_sec: 15.0\n enable_cpu_profiler: true\n enable_trial_buffer_preparation: false\n enforce_publisher_rate: false\n input_data_end_time: -1.0\n input_data_path: datasets/r2b_dataset/r2b_storage\n input_data_start_time: -1.0\n linear_scan_acceptable_frame_loss_fraction: 0.05\n linear_scan_acceptable_frame_rate_drop: 5.0\n linear_scan_duration_fraction: 1.0\n linear_scan_step_size: 5.0\n load_data_in_real_time: false\n log_file_name: kpi.json\n log_folder: /osmo/data/output\n monitor_info_list:\n - calculators:\n - class_name: BasicPerformanceCalculator\n config:\n report_prefix: ''\n module_name: ros2_benchmark.basic_performance_calculator\n service_name: start_monitoring0\n play_messages_service_future_timeout_sec: 15.0\n playback_message_buffer_size: 100\n publish_tf_messages_in_set_data: false\n publish_tf_static_messages_in_set_data: false\n publisher_lower_frequency: 10.0\n publisher_upper_frequency: 2500.0\n record_data_timeline: false\n revise_timestamps_as_message_ids: false\n set_data_service_future_timeout_sec: 60.0\n setup_service_client_timeout_sec: 20.0\n start_monitoring_service_timeout_sec: 10\n start_recording_service_future_timeout_sec: 35.0\n start_recording_service_timeout_sec: 30\n test_iterations: 5\n" - } -} \ No newline at end of file diff --git a/results/image_transport_h264_decoder_node-agx_orin.json b/results/image_transport_h264_decoder_node-agx_orin.json deleted file mode 100644 index 13295c8..0000000 --- a/results/image_transport_h264_decoder_node-agx_orin.json +++ /dev/null @@ -1,51 +0,0 @@ -{ - "BasicPerformanceMetrics.RECEIVED_DURATION": 4961.733642578125, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 28.013717214012107, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 28.01440182313297, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0.0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 139.0, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 36.534749348958336, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 36.396158854166664, - "BasicPerformanceMetrics.MAX_JITTER": 2.666259765625, - "BasicPerformanceMetrics.MIN_JITTER": 0.000244140625, - "BasicPerformanceMetrics.MEAN_JITTER": 0.17790188754562045, - "BasicPerformanceMetrics.STD_DEV_JITTER": 0.1734564983908401, - "CPUProfilingMetrics.MAX_CPU_UTIL": 10.35, - "CPUProfilingMetrics.MIN_CPU_UTIL": 8.825000000000001, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 9.449444444444444, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 0.3942085073673369, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 9.630555555555555, - "10.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4899.842041015625, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 10.203956564228177, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 10.204410587414802, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 50, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 36.57470703125, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 36.356689453125, - "BasicPerformanceMetrics.MAX_JITTER": 0.42529296875, - "BasicPerformanceMetrics.MIN_JITTER": 0.015869140625, - "BasicPerformanceMetrics.MEAN_JITTER": 0.1478729248046875, - "BasicPerformanceMetrics.STD_DEV_JITTER": 0.11037473132949785, - "CPUProfilingMetrics.MAX_CPU_UTIL": 3.966666666666667, - "CPUProfilingMetrics.MIN_CPU_UTIL": 3.5416666666666665, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 3.7416666666666663, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 0.17440374613713633, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 3.933333333333333 - }, - "metadata": { - "BenchmarkMetadata.NAME": "image_transport H264 Decoder Node Benchmark", - "BenchmarkMetadata.TEST_FILE_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/scripts/image_transport_h264_decoder_node.py", - "BenchmarkMetadata.TEST_DATETIME": "2023-05-25T06:45:56Z", - "BenchmarkMetadata.DEVICE_HOSTNAME": "ros-fmln7xgkgfa5zpisqn7gfxy5ki", - "BenchmarkMetadata.DEVICE_ARCH": "aarch64", - "BenchmarkMetadata.DEVICE_OS": "Linux 5.10.104-tegra #1 SMP PREEMPT Sun Mar 19 07:55:28 PDT 2023", - "BenchmarkMetadata.IDLE_CPU_UTIL": 0.25, - "BenchmarkMetadata.BENCHMARK_MODE": 1, - "BenchmarkMetadata.PEAK_THROUGHPUT_PREDICTION": 27.8125, - "BenchmarkMetadata.INPUT_DATA_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets/datasets/r2b_dataset/r2b_mezzanine", - "BenchmarkMetadata.INPUT_DATA_SIZE": 2057437184, - "BenchmarkMetadata.INPUT_DATA_HASH": "c84d7420c5524d882f1019d350232fb1", - "BenchmarkMetadata.CONFIG": "ros2_benchmark_config:\n additional_fixed_publisher_rate_tests:\n - 10.0\n - 30.0\n - 60.0\n assets_root: /workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets\n benchmark_duration: 5.0\n benchmark_mode: BenchmarkMode.LOOPING\n benchmark_name: image_transport H264 Decoder Node Benchmark\n benchmark_namespace: r2b\n binary_search_acceptable_frame_loss_fraction: 0.05\n binary_search_acceptable_frame_rate_drop: 10.0\n binary_search_duration_fraction: 1.0\n binary_search_terminal_interval_width: 10.0\n cpu_profiling_interval_sec: 1.0\n custom_report_info: {}\n default_config_version: 0.30.0\n default_service_future_timeout_sec: 15.0\n enable_cpu_profiler: true\n enable_trial_buffer_preparation: false\n enforce_publisher_rate: false\n input_data_end_time: -1.0\n input_data_path: datasets/r2b_dataset/r2b_mezzanine\n input_data_start_time: -1.0\n linear_scan_acceptable_frame_loss_fraction: 0.05\n linear_scan_acceptable_frame_rate_drop: 5.0\n linear_scan_duration_fraction: 1.0\n linear_scan_step_size: 5.0\n load_data_in_real_time: false\n log_file_name: kpi.json\n log_folder: /osmo/data/output\n monitor_info_list:\n - calculators:\n - class_name: BasicPerformanceCalculator\n config:\n report_prefix: ''\n module_name: ros2_benchmark.basic_performance_calculator\n service_name: start_monitoring0\n play_messages_service_future_timeout_sec: 15.0\n playback_message_buffer_size: 1\n publish_tf_messages_in_set_data: false\n publish_tf_static_messages_in_set_data: false\n publisher_lower_frequency: 10.0\n publisher_upper_frequency: 200.0\n record_data_timeline: false\n revise_timestamps_as_message_ids: false\n set_data_service_future_timeout_sec: 60.0\n setup_service_client_timeout_sec: 20.0\n start_monitoring_service_timeout_sec: 10\n start_recording_service_future_timeout_sec: 35.0\n start_recording_service_timeout_sec: 30\n test_iterations: 5\n" - } -} \ No newline at end of file diff --git a/results/image_transport_h264_decoder_node-agx_xavier.json b/results/image_transport_h264_decoder_node-agx_xavier.json deleted file mode 100644 index 4d10037..0000000 --- a/results/image_transport_h264_decoder_node-agx_xavier.json +++ /dev/null @@ -1,51 +0,0 @@ -{ - "BasicPerformanceMetrics.RECEIVED_DURATION": 5347.660237630208, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 28.01324582001166, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 25.244782543113505, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 4.0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 139.0, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 42.08642578125, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 427.9322102864583, - "BasicPerformanceMetrics.MAX_JITTER": 18.798828125, - "BasicPerformanceMetrics.MIN_JITTER": 0.00341796875, - "BasicPerformanceMetrics.MEAN_JITTER": 1.1705698572603382, - "BasicPerformanceMetrics.STD_DEV_JITTER": 2.9368290708684044, - "CPUProfilingMetrics.MAX_CPU_UTIL": 18.0, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.75, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 9.56625, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 7.106326975476958, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 15.104166666666666, - "10.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4897.954345703125, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 10.203881829681567, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 10.208343416647804, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 50, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 43.230224609375, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 41.088623046875, - "BasicPerformanceMetrics.MAX_JITTER": 17.03076171875, - "BasicPerformanceMetrics.MIN_JITTER": 0.095947265625, - "BasicPerformanceMetrics.MEAN_JITTER": 2.64202880859375, - "BasicPerformanceMetrics.STD_DEV_JITTER": 3.4756738435983383, - "CPUProfilingMetrics.MAX_CPU_UTIL": 8.75, - "CPUProfilingMetrics.MIN_CPU_UTIL": 6.625, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 7.695, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 0.8099614188342553, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 7.2625 - }, - "metadata": { - "BenchmarkMetadata.NAME": "image_transport H264 Decoder Node Benchmark", - "BenchmarkMetadata.TEST_FILE_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/scripts/image_transport_h264_decoder_node.py", - "BenchmarkMetadata.TEST_DATETIME": "2023-05-25T07:35:09Z", - "BenchmarkMetadata.DEVICE_HOSTNAME": "ros-yydzlnzf2rgd7fzb7vxq2fd32q", - "BenchmarkMetadata.DEVICE_ARCH": "aarch64", - "BenchmarkMetadata.DEVICE_OS": "Linux 5.10.104-tegra #1 SMP PREEMPT Sun Mar 19 07:55:28 PDT 2023", - "BenchmarkMetadata.IDLE_CPU_UTIL": 2.3625, - "BenchmarkMetadata.BENCHMARK_MODE": 1, - "BenchmarkMetadata.PEAK_THROUGHPUT_PREDICTION": 27.8125, - "BenchmarkMetadata.INPUT_DATA_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets/datasets/r2b_dataset/r2b_mezzanine", - "BenchmarkMetadata.INPUT_DATA_SIZE": 2057437184, - "BenchmarkMetadata.INPUT_DATA_HASH": "c84d7420c5524d882f1019d350232fb1", - "BenchmarkMetadata.CONFIG": "ros2_benchmark_config:\n additional_fixed_publisher_rate_tests:\n - 10.0\n - 30.0\n - 60.0\n assets_root: /workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets\n benchmark_duration: 5.0\n benchmark_mode: BenchmarkMode.LOOPING\n benchmark_name: image_transport H264 Decoder Node Benchmark\n benchmark_namespace: r2b\n binary_search_acceptable_frame_loss_fraction: 0.05\n binary_search_acceptable_frame_rate_drop: 10.0\n binary_search_duration_fraction: 1.0\n binary_search_terminal_interval_width: 10.0\n cpu_profiling_interval_sec: 1.0\n custom_report_info: {}\n default_config_version: 0.30.0\n default_service_future_timeout_sec: 15.0\n enable_cpu_profiler: true\n enable_trial_buffer_preparation: false\n enforce_publisher_rate: false\n input_data_end_time: -1.0\n input_data_path: datasets/r2b_dataset/r2b_mezzanine\n input_data_start_time: -1.0\n linear_scan_acceptable_frame_loss_fraction: 0.05\n linear_scan_acceptable_frame_rate_drop: 5.0\n linear_scan_duration_fraction: 1.0\n linear_scan_step_size: 5.0\n load_data_in_real_time: false\n log_file_name: kpi.json\n log_folder: /osmo/data/output\n monitor_info_list:\n - calculators:\n - class_name: BasicPerformanceCalculator\n config:\n report_prefix: ''\n module_name: ros2_benchmark.basic_performance_calculator\n service_name: start_monitoring0\n play_messages_service_future_timeout_sec: 15.0\n playback_message_buffer_size: 1\n publish_tf_messages_in_set_data: false\n publish_tf_static_messages_in_set_data: false\n publisher_lower_frequency: 10.0\n publisher_upper_frequency: 200.0\n record_data_timeline: false\n revise_timestamps_as_message_ids: false\n set_data_service_future_timeout_sec: 60.0\n setup_service_client_timeout_sec: 20.0\n start_monitoring_service_timeout_sec: 10\n start_recording_service_future_timeout_sec: 35.0\n start_recording_service_timeout_sec: 30\n test_iterations: 5\n" - } -} \ No newline at end of file diff --git a/results/image_transport_h264_decoder_node-nuc_3060ti.json b/results/image_transport_h264_decoder_node-nuc_3060ti.json deleted file mode 100644 index 4da7f92..0000000 --- a/results/image_transport_h264_decoder_node-nuc_3060ti.json +++ /dev/null @@ -1,87 +0,0 @@ -{ - "BasicPerformanceMetrics.RECEIVED_DURATION": 5144.278483072917, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 63.63701057727841, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 60.131497341954685, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 8.0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 317.0, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 22.668619791666668, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 185.57421875, - "BasicPerformanceMetrics.MAX_JITTER": 19.93896484375, - "BasicPerformanceMetrics.MIN_JITTER": 0.000244140625, - "BasicPerformanceMetrics.MEAN_JITTER": 3.140176481066775, - "BasicPerformanceMetrics.STD_DEV_JITTER": 5.3269592708557, - "CPUProfilingMetrics.MAX_CPU_UTIL": 7.5625, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.1875, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 3.578219696969697, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 3.3064556693942335, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 7.145833333333333, - "10.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4923.877197265625, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 10.203372443769977, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 10.154599312055646, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 50, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 19.671630859375, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 43.208251953125, - "BasicPerformanceMetrics.MAX_JITTER": 24.660400390625, - "BasicPerformanceMetrics.MIN_JITTER": 0.05712890625, - "BasicPerformanceMetrics.MEAN_JITTER": 4.4874827067057295, - "BasicPerformanceMetrics.STD_DEV_JITTER": 6.456146213761829, - "CPUProfilingMetrics.MAX_CPU_UTIL": 2.8874999999999997, - "CPUProfilingMetrics.MIN_CPU_UTIL": 2.0, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 2.44375, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 0.28758151018450395, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 2.51875 - }, - "60.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 5012.2587890625, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 60.1991215278584, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 59.853254316126886, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 300, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 19.713134765625, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 48.510498046875, - "BasicPerformanceMetrics.MAX_JITTER": 19.724365234375, - "BasicPerformanceMetrics.MIN_JITTER": 0.000244140625, - "BasicPerformanceMetrics.MEAN_JITTER": 2.9416348246119965, - "BasicPerformanceMetrics.STD_DEV_JITTER": 5.433858457281268, - "CPUProfilingMetrics.MAX_CPU_UTIL": 7.5, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.6875, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 6.145833333333333, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 2.4440794392890663, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 7.125 - }, - "30.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4977.31982421875, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 30.199867678965184, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 30.136701135845595, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 150, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 18.97998046875, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 29.390625, - "BasicPerformanceMetrics.MAX_JITTER": 24.01025390625, - "BasicPerformanceMetrics.MIN_JITTER": 0.01806640625, - "BasicPerformanceMetrics.MEAN_JITTER": 3.06365306957348, - "BasicPerformanceMetrics.STD_DEV_JITTER": 5.346445996670742, - "CPUProfilingMetrics.MAX_CPU_UTIL": 4.36875, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.25, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 3.502083333333333, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 1.4652825500185582, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 4.36875 - }, - "metadata": { - "BenchmarkMetadata.NAME": "image_transport H264 Decoder Node Benchmark", - "BenchmarkMetadata.TEST_FILE_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/scripts/image_transport_h264_decoder_node.py", - "BenchmarkMetadata.TEST_DATETIME": "2023-05-25T08:18:50Z", - "BenchmarkMetadata.DEVICE_HOSTNAME": "ros-23yloy6235asze6nmcetl3m7te", - "BenchmarkMetadata.DEVICE_ARCH": "x86_64", - "BenchmarkMetadata.DEVICE_OS": "Linux 5.15.0-50-generic #56~20.04.1-Ubuntu SMP Tue Sep 27 15:51:29 UTC 2022", - "BenchmarkMetadata.IDLE_CPU_UTIL": 0.875, - "BenchmarkMetadata.BENCHMARK_MODE": 1, - "BenchmarkMetadata.PEAK_THROUGHPUT_PREDICTION": 63.4375, - "BenchmarkMetadata.INPUT_DATA_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets/datasets/r2b_dataset/r2b_mezzanine", - "BenchmarkMetadata.INPUT_DATA_SIZE": 2057437184, - "BenchmarkMetadata.INPUT_DATA_HASH": "c84d7420c5524d882f1019d350232fb1", - "BenchmarkMetadata.CONFIG": "ros2_benchmark_config:\n additional_fixed_publisher_rate_tests:\n - 10.0\n - 30.0\n - 60.0\n assets_root: /workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets\n benchmark_duration: 5.0\n benchmark_mode: BenchmarkMode.LOOPING\n benchmark_name: image_transport H264 Decoder Node Benchmark\n benchmark_namespace: r2b\n binary_search_acceptable_frame_loss_fraction: 0.05\n binary_search_acceptable_frame_rate_drop: 10.0\n binary_search_duration_fraction: 1.0\n binary_search_terminal_interval_width: 10.0\n cpu_profiling_interval_sec: 1.0\n custom_report_info: {}\n default_config_version: 0.30.0\n default_service_future_timeout_sec: 15.0\n enable_cpu_profiler: true\n enable_trial_buffer_preparation: false\n enforce_publisher_rate: false\n input_data_end_time: -1.0\n input_data_path: datasets/r2b_dataset/r2b_mezzanine\n input_data_start_time: -1.0\n linear_scan_acceptable_frame_loss_fraction: 0.05\n linear_scan_acceptable_frame_rate_drop: 5.0\n linear_scan_duration_fraction: 1.0\n linear_scan_step_size: 5.0\n load_data_in_real_time: false\n log_file_name: kpi.json\n log_folder: /osmo/data/output\n monitor_info_list:\n - calculators:\n - class_name: BasicPerformanceCalculator\n config:\n report_prefix: ''\n module_name: ros2_benchmark.basic_performance_calculator\n service_name: start_monitoring0\n play_messages_service_future_timeout_sec: 15.0\n playback_message_buffer_size: 1\n publish_tf_messages_in_set_data: false\n publish_tf_static_messages_in_set_data: false\n publisher_lower_frequency: 10.0\n publisher_upper_frequency: 200.0\n record_data_timeline: false\n revise_timestamps_as_message_ids: false\n set_data_service_future_timeout_sec: 60.0\n setup_service_client_timeout_sec: 20.0\n start_monitoring_service_timeout_sec: 10\n start_recording_service_future_timeout_sec: 35.0\n start_recording_service_timeout_sec: 30\n test_iterations: 5\n" - } -} \ No newline at end of file diff --git a/results/image_transport_h264_decoder_node-nuc_4060ti.json b/results/image_transport_h264_decoder_node-nuc_4060ti.json deleted file mode 100644 index a948782..0000000 --- a/results/image_transport_h264_decoder_node-nuc_4060ti.json +++ /dev/null @@ -1,87 +0,0 @@ -{ - "BasicPerformanceMetrics.RECEIVED_DURATION": 5143.83056640625, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 63.63686918763812, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 60.49516309565987, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 5.666666666666667, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 317.0, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 22.2666015625, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 184.76652018229166, - "BasicPerformanceMetrics.MAX_JITTER": 21.709228515625, - "BasicPerformanceMetrics.MIN_JITTER": 0.000244140625, - "BasicPerformanceMetrics.MEAN_JITTER": 3.3547473538306454, - "BasicPerformanceMetrics.STD_DEV_JITTER": 5.734073683562918, - "CPUProfilingMetrics.MAX_CPU_UTIL": 7.55625, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.0625, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 3.559090909090909, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 3.26819524582408, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 7.14375, - "10.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4914.766357421875, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 10.203432428690437, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 10.173423590013412, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 50, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 18.675048828125, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 33.129638671875, - "BasicPerformanceMetrics.MAX_JITTER": 26.2314453125, - "BasicPerformanceMetrics.MIN_JITTER": 0.03369140625, - "BasicPerformanceMetrics.MEAN_JITTER": 4.114115397135417, - "BasicPerformanceMetrics.STD_DEV_JITTER": 6.1817740863041335, - "CPUProfilingMetrics.MAX_CPU_UTIL": 2.66875, - "CPUProfilingMetrics.MIN_CPU_UTIL": 2.275, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 2.48875, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 0.1313630465541966, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 2.66875 - }, - "60.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4992.79443359375, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 60.19940464905187, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 60.086591585158416, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 300, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 18.781982421875, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 28.138427734375, - "BasicPerformanceMetrics.MAX_JITTER": 19.7548828125, - "BasicPerformanceMetrics.MIN_JITTER": 0.00390625, - "BasicPerformanceMetrics.MEAN_JITTER": 2.362466901740772, - "BasicPerformanceMetrics.STD_DEV_JITTER": 4.956252622826099, - "CPUProfilingMetrics.MAX_CPU_UTIL": 7.48125, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.6875, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 6.037499999999999, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 2.40070247965743, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 7.1875 - }, - "30.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4968.34423828125, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 30.20082219575609, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 30.191144736760638, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 150, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 18.581787109375, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 20.173828125, - "BasicPerformanceMetrics.MAX_JITTER": 21.677734375, - "BasicPerformanceMetrics.MIN_JITTER": 0.012939453125, - "BasicPerformanceMetrics.MEAN_JITTER": 2.4685800913217903, - "BasicPerformanceMetrics.STD_DEV_JITTER": 4.4518960778811865, - "CPUProfilingMetrics.MAX_CPU_UTIL": 4.3375, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.125, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 3.4031249999999997, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 1.4759608011252647, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 4.0625 - }, - "metadata": { - "BenchmarkMetadata.NAME": "image_transport H264 Decoder Node Benchmark", - "BenchmarkMetadata.TEST_FILE_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/scripts/image_transport_h264_decoder_node.py", - "BenchmarkMetadata.TEST_DATETIME": "2023-05-25T08:12:26Z", - "BenchmarkMetadata.DEVICE_HOSTNAME": "ros-7dg4ybjkefecpkzqnndhqetfkq", - "BenchmarkMetadata.DEVICE_ARCH": "x86_64", - "BenchmarkMetadata.DEVICE_OS": "Linux 5.15.0-72-generic #79~20.04.1-Ubuntu SMP Thu Apr 20 22:12:07 UTC 2023", - "BenchmarkMetadata.IDLE_CPU_UTIL": 0.375, - "BenchmarkMetadata.BENCHMARK_MODE": 1, - "BenchmarkMetadata.PEAK_THROUGHPUT_PREDICTION": 63.4375, - "BenchmarkMetadata.INPUT_DATA_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets/datasets/r2b_dataset/r2b_mezzanine", - "BenchmarkMetadata.INPUT_DATA_SIZE": 2057437184, - "BenchmarkMetadata.INPUT_DATA_HASH": "c84d7420c5524d882f1019d350232fb1", - "BenchmarkMetadata.CONFIG": "ros2_benchmark_config:\n additional_fixed_publisher_rate_tests:\n - 10.0\n - 30.0\n - 60.0\n assets_root: /workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets\n benchmark_duration: 5.0\n benchmark_mode: BenchmarkMode.LOOPING\n benchmark_name: image_transport H264 Decoder Node Benchmark\n benchmark_namespace: r2b\n binary_search_acceptable_frame_loss_fraction: 0.05\n binary_search_acceptable_frame_rate_drop: 10.0\n binary_search_duration_fraction: 1.0\n binary_search_terminal_interval_width: 10.0\n cpu_profiling_interval_sec: 1.0\n custom_report_info: {}\n default_config_version: 0.30.0\n default_service_future_timeout_sec: 15.0\n enable_cpu_profiler: true\n enable_trial_buffer_preparation: false\n enforce_publisher_rate: false\n input_data_end_time: -1.0\n input_data_path: datasets/r2b_dataset/r2b_mezzanine\n input_data_start_time: -1.0\n linear_scan_acceptable_frame_loss_fraction: 0.05\n linear_scan_acceptable_frame_rate_drop: 5.0\n linear_scan_duration_fraction: 1.0\n linear_scan_step_size: 5.0\n load_data_in_real_time: false\n log_file_name: kpi.json\n log_folder: /osmo/data/output\n monitor_info_list:\n - calculators:\n - class_name: BasicPerformanceCalculator\n config:\n report_prefix: ''\n module_name: ros2_benchmark.basic_performance_calculator\n service_name: start_monitoring0\n play_messages_service_future_timeout_sec: 15.0\n playback_message_buffer_size: 1\n publish_tf_messages_in_set_data: false\n publish_tf_static_messages_in_set_data: false\n publisher_lower_frequency: 10.0\n publisher_upper_frequency: 200.0\n record_data_timeline: false\n revise_timestamps_as_message_ids: false\n set_data_service_future_timeout_sec: 60.0\n setup_service_client_timeout_sec: 20.0\n start_monitoring_service_timeout_sec: 10\n start_recording_service_future_timeout_sec: 35.0\n start_recording_service_timeout_sec: 30\n test_iterations: 5\n" - } -} \ No newline at end of file diff --git a/results/image_transport_h264_decoder_node-orin_nx.json b/results/image_transport_h264_decoder_node-orin_nx.json deleted file mode 100644 index 63788b5..0000000 --- a/results/image_transport_h264_decoder_node-orin_nx.json +++ /dev/null @@ -1,51 +0,0 @@ -{ - "BasicPerformanceMetrics.RECEIVED_DURATION": 5338.857259114583, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 28.013451650525017, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 25.786447701400984, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 1.3333333333333333, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 139.0, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 40.575113932291664, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 417.5304361979167, - "BasicPerformanceMetrics.MAX_JITTER": 5.83251953125, - "BasicPerformanceMetrics.MIN_JITTER": 0.00048828125, - "BasicPerformanceMetrics.MEAN_JITTER": 0.1617802372685185, - "BasicPerformanceMetrics.STD_DEV_JITTER": 0.34383075412454484, - "CPUProfilingMetrics.MAX_CPU_UTIL": 14.875, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.5, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 8.075416666666667, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 6.479360465709649, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 14.737499999999999, - "10.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4899.837890625, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 10.203895047957062, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 10.20441923102526, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 50, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 40.648193359375, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 40.396484375, - "BasicPerformanceMetrics.MAX_JITTER": 0.453125, - "BasicPerformanceMetrics.MIN_JITTER": 0.002197265625, - "BasicPerformanceMetrics.MEAN_JITTER": 0.13733927408854166, - "BasicPerformanceMetrics.STD_DEV_JITTER": 0.09234267443324408, - "CPUProfilingMetrics.MAX_CPU_UTIL": 6.925, - "CPUProfilingMetrics.MIN_CPU_UTIL": 5.3875, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 6.09, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 0.5078385570237848, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 6.925 - }, - "metadata": { - "BenchmarkMetadata.NAME": "image_transport H264 Decoder Node Benchmark", - "BenchmarkMetadata.TEST_FILE_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/scripts/image_transport_h264_decoder_node.py", - "BenchmarkMetadata.TEST_DATETIME": "2023-05-25T06:14:05Z", - "BenchmarkMetadata.DEVICE_HOSTNAME": "ros-purco3dir5es7gbopwdymfxu7y", - "BenchmarkMetadata.DEVICE_ARCH": "aarch64", - "BenchmarkMetadata.DEVICE_OS": "Linux 5.10.104-tegra #1 SMP PREEMPT Wed Mar 15 00:17:07 PDT 2023", - "BenchmarkMetadata.IDLE_CPU_UTIL": 0.5, - "BenchmarkMetadata.BENCHMARK_MODE": 1, - "BenchmarkMetadata.PEAK_THROUGHPUT_PREDICTION": 27.8125, - "BenchmarkMetadata.INPUT_DATA_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets/datasets/r2b_dataset/r2b_mezzanine", - "BenchmarkMetadata.INPUT_DATA_SIZE": 2057437184, - "BenchmarkMetadata.INPUT_DATA_HASH": "c84d7420c5524d882f1019d350232fb1", - "BenchmarkMetadata.CONFIG": "ros2_benchmark_config:\n additional_fixed_publisher_rate_tests:\n - 10.0\n - 30.0\n - 60.0\n assets_root: /workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets\n benchmark_duration: 5.0\n benchmark_mode: BenchmarkMode.LOOPING\n benchmark_name: image_transport H264 Decoder Node Benchmark\n benchmark_namespace: r2b\n binary_search_acceptable_frame_loss_fraction: 0.05\n binary_search_acceptable_frame_rate_drop: 10.0\n binary_search_duration_fraction: 1.0\n binary_search_terminal_interval_width: 10.0\n cpu_profiling_interval_sec: 1.0\n custom_report_info: {}\n default_config_version: 0.30.0\n default_service_future_timeout_sec: 15.0\n enable_cpu_profiler: true\n enable_trial_buffer_preparation: false\n enforce_publisher_rate: false\n input_data_end_time: -1.0\n input_data_path: datasets/r2b_dataset/r2b_mezzanine\n input_data_start_time: -1.0\n linear_scan_acceptable_frame_loss_fraction: 0.05\n linear_scan_acceptable_frame_rate_drop: 5.0\n linear_scan_duration_fraction: 1.0\n linear_scan_step_size: 5.0\n load_data_in_real_time: false\n log_file_name: kpi.json\n log_folder: /osmo/data/output\n monitor_info_list:\n - calculators:\n - class_name: BasicPerformanceCalculator\n config:\n report_prefix: ''\n module_name: ros2_benchmark.basic_performance_calculator\n service_name: start_monitoring0\n play_messages_service_future_timeout_sec: 15.0\n playback_message_buffer_size: 1\n publish_tf_messages_in_set_data: false\n publish_tf_static_messages_in_set_data: false\n publisher_lower_frequency: 10.0\n publisher_upper_frequency: 200.0\n record_data_timeline: false\n revise_timestamps_as_message_ids: false\n set_data_service_future_timeout_sec: 60.0\n setup_service_client_timeout_sec: 20.0\n start_monitoring_service_timeout_sec: 10\n start_recording_service_future_timeout_sec: 35.0\n start_recording_service_timeout_sec: 30\n test_iterations: 5\n" - } -} \ No newline at end of file diff --git a/results/image_transport_h264_decoder_node-x86_4090.json b/results/image_transport_h264_decoder_node-x86_4090.json deleted file mode 100644 index cf478ae..0000000 --- a/results/image_transport_h264_decoder_node-x86_4090.json +++ /dev/null @@ -1,69 +0,0 @@ -{ - "BasicPerformanceMetrics.RECEIVED_DURATION": 5134.246012369792, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 56.763034900653984, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 52.60024431935014, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 11.666666666666666, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 282.0, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 33.992106119791664, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 198.77799479166666, - "BasicPerformanceMetrics.MAX_JITTER": 26.4140625, - "BasicPerformanceMetrics.MIN_JITTER": 0.0, - "BasicPerformanceMetrics.MEAN_JITTER": 5.034435539667896, - "BasicPerformanceMetrics.STD_DEV_JITTER": 3.4458721847453604, - "CPUProfilingMetrics.MAX_CPU_UTIL": 8.15625, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.1875, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 3.8409090909090913, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 3.431069277573003, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 7.860416666666667, - "10.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4889.73193359375, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 10.203848275751426, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 10.225509430585916, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 50, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 30.84228515625, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 20.462158203125, - "BasicPerformanceMetrics.MAX_JITTER": 23.059814453125, - "BasicPerformanceMetrics.MIN_JITTER": 0.078369140625, - "BasicPerformanceMetrics.MEAN_JITTER": 7.9818166097005205, - "BasicPerformanceMetrics.STD_DEV_JITTER": 5.935515289329209, - "CPUProfilingMetrics.MAX_CPU_UTIL": 2.8625, - "CPUProfilingMetrics.MIN_CPU_UTIL": 1.425, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 1.9962499999999999, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 0.4825971404805461, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 1.99375 - }, - "30.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4963.3818359375, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 30.20118590776487, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 30.22132992346488, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 150, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 27.913818359375, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 24.603271484375, - "BasicPerformanceMetrics.MAX_JITTER": 24.06494140625, - "BasicPerformanceMetrics.MIN_JITTER": 0.016845703125, - "BasicPerformanceMetrics.MEAN_JITTER": 4.134463748416385, - "BasicPerformanceMetrics.STD_DEV_JITTER": 4.962593669693698, - "CPUProfilingMetrics.MAX_CPU_UTIL": 5.781250000000001, - "CPUProfilingMetrics.MIN_CPU_UTIL": 4.85625, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 5.2725, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 0.34391132578035305, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 5.781250000000001 - }, - "metadata": { - "BenchmarkMetadata.NAME": "image_transport H264 Decoder Node Benchmark", - "BenchmarkMetadata.TEST_FILE_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/scripts/image_transport_h264_decoder_node.py", - "BenchmarkMetadata.TEST_DATETIME": "2023-05-25T06:48:10Z", - "BenchmarkMetadata.DEVICE_HOSTNAME": "ros-r3wzloa6xjcrbc6va35rja3cce", - "BenchmarkMetadata.DEVICE_ARCH": "x86_64", - "BenchmarkMetadata.DEVICE_OS": "Linux 5.4.0-144-generic #161-Ubuntu SMP Fri Feb 3 14:49:04 UTC 2023", - "BenchmarkMetadata.IDLE_CPU_UTIL": 0.1875, - "BenchmarkMetadata.BENCHMARK_MODE": 1, - "BenchmarkMetadata.PEAK_THROUGHPUT_PREDICTION": 56.5625, - "BenchmarkMetadata.INPUT_DATA_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets/datasets/r2b_dataset/r2b_mezzanine", - "BenchmarkMetadata.INPUT_DATA_SIZE": 2057437184, - "BenchmarkMetadata.INPUT_DATA_HASH": "c84d7420c5524d882f1019d350232fb1", - "BenchmarkMetadata.CONFIG": "ros2_benchmark_config:\n additional_fixed_publisher_rate_tests:\n - 10.0\n - 30.0\n - 60.0\n assets_root: /workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets\n benchmark_duration: 5.0\n benchmark_mode: BenchmarkMode.LOOPING\n benchmark_name: image_transport H264 Decoder Node Benchmark\n benchmark_namespace: r2b\n binary_search_acceptable_frame_loss_fraction: 0.05\n binary_search_acceptable_frame_rate_drop: 10.0\n binary_search_duration_fraction: 1.0\n binary_search_terminal_interval_width: 10.0\n cpu_profiling_interval_sec: 1.0\n custom_report_info: {}\n default_config_version: 0.30.0\n default_service_future_timeout_sec: 15.0\n enable_cpu_profiler: true\n enable_trial_buffer_preparation: false\n enforce_publisher_rate: false\n input_data_end_time: -1.0\n input_data_path: datasets/r2b_dataset/r2b_mezzanine\n input_data_start_time: -1.0\n linear_scan_acceptable_frame_loss_fraction: 0.05\n linear_scan_acceptable_frame_rate_drop: 5.0\n linear_scan_duration_fraction: 1.0\n linear_scan_step_size: 5.0\n load_data_in_real_time: false\n log_file_name: kpi.json\n log_folder: /osmo/data/output\n monitor_info_list:\n - calculators:\n - class_name: BasicPerformanceCalculator\n config:\n report_prefix: ''\n module_name: ros2_benchmark.basic_performance_calculator\n service_name: start_monitoring0\n play_messages_service_future_timeout_sec: 15.0\n playback_message_buffer_size: 1\n publish_tf_messages_in_set_data: false\n publish_tf_static_messages_in_set_data: false\n publisher_lower_frequency: 10.0\n publisher_upper_frequency: 200.0\n record_data_timeline: false\n revise_timestamps_as_message_ids: false\n set_data_service_future_timeout_sec: 60.0\n setup_service_client_timeout_sec: 20.0\n start_monitoring_service_timeout_sec: 10\n start_recording_service_future_timeout_sec: 35.0\n start_recording_service_timeout_sec: 30\n test_iterations: 5\n" - } -} \ No newline at end of file diff --git a/results/image_transport_h264_encoder_node-agx_orin.json b/results/image_transport_h264_encoder_node-agx_orin.json deleted file mode 100644 index 86d53b0..0000000 --- a/results/image_transport_h264_encoder_node-agx_orin.json +++ /dev/null @@ -1,51 +0,0 @@ -{ - "BasicPerformanceMetrics.RECEIVED_DURATION": 4900.6611328125, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 10.20406129635504, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 10.202705039179907, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0.0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 50.0, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 94.56193033854167, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 95.23697916666667, - "BasicPerformanceMetrics.MAX_JITTER": 4.053466796875, - "BasicPerformanceMetrics.MIN_JITTER": 0.003173828125, - "BasicPerformanceMetrics.MEAN_JITTER": 0.5315450032552084, - "BasicPerformanceMetrics.STD_DEV_JITTER": 0.7194135062606462, - "CPUProfilingMetrics.MAX_CPU_UTIL": 9.716666666666667, - "CPUProfilingMetrics.MIN_CPU_UTIL": 8.25, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 8.859444444444446, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 0.3499393617119409, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 9.055555555555557, - "10.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4890.792724609375, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 10.204273816939512, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 10.223291563433301, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 50, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 104.265869140625, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 95.15087890625, - "BasicPerformanceMetrics.MAX_JITTER": 3.2333984375, - "BasicPerformanceMetrics.MIN_JITTER": 0.006103515625, - "BasicPerformanceMetrics.MEAN_JITTER": 0.4900614420572917, - "BasicPerformanceMetrics.STD_DEV_JITTER": 0.7124865496463395, - "CPUProfilingMetrics.MAX_CPU_UTIL": 8.841666666666667, - "CPUProfilingMetrics.MIN_CPU_UTIL": 8.0, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 8.370000000000001, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 0.28395030864180765, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 8.841666666666667 - }, - "metadata": { - "BenchmarkMetadata.NAME": "image_transport H264 Encoder Node Benchmark", - "BenchmarkMetadata.TEST_FILE_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/scripts/image_transport_h264_encoder_node.py", - "BenchmarkMetadata.TEST_DATETIME": "2023-05-25T07:08:54Z", - "BenchmarkMetadata.DEVICE_HOSTNAME": "ros-uno5646aejg4jenoj2ixcxtncu", - "BenchmarkMetadata.DEVICE_ARCH": "aarch64", - "BenchmarkMetadata.DEVICE_OS": "Linux 5.10.104-tegra #1 SMP PREEMPT Sun Mar 19 07:55:28 PDT 2023", - "BenchmarkMetadata.IDLE_CPU_UTIL": 0.75, - "BenchmarkMetadata.BENCHMARK_MODE": 1, - "BenchmarkMetadata.PEAK_THROUGHPUT_PREDICTION": 10.0, - "BenchmarkMetadata.INPUT_DATA_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets/datasets/r2b_dataset/r2b_mezzanine", - "BenchmarkMetadata.INPUT_DATA_SIZE": 2057437184, - "BenchmarkMetadata.INPUT_DATA_HASH": "c84d7420c5524d882f1019d350232fb1", - "BenchmarkMetadata.CONFIG": "ros2_benchmark_config:\n additional_fixed_publisher_rate_tests:\n - 10.0\n - 30.0\n - 60.0\n assets_root: /workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets\n benchmark_duration: 5.0\n benchmark_mode: BenchmarkMode.LOOPING\n benchmark_name: image_transport H264 Encoder Node Benchmark\n benchmark_namespace: r2b\n binary_search_acceptable_frame_loss_fraction: 0.05\n binary_search_acceptable_frame_rate_drop: 10.0\n binary_search_duration_fraction: 1.0\n binary_search_terminal_interval_width: 10.0\n cpu_profiling_interval_sec: 1.0\n custom_report_info: {}\n default_config_version: 0.30.0\n default_service_future_timeout_sec: 15.0\n enable_cpu_profiler: true\n enable_trial_buffer_preparation: false\n enforce_publisher_rate: false\n input_data_end_time: -1.0\n input_data_path: datasets/r2b_dataset/r2b_mezzanine\n input_data_start_time: -1.0\n linear_scan_acceptable_frame_loss_fraction: 0.05\n linear_scan_acceptable_frame_rate_drop: 5.0\n linear_scan_duration_fraction: 1.0\n linear_scan_step_size: 5.0\n load_data_in_real_time: false\n log_file_name: kpi.json\n log_folder: /osmo/data/output\n monitor_info_list:\n - calculators:\n - class_name: BasicPerformanceCalculator\n config:\n report_prefix: ''\n module_name: ros2_benchmark.basic_performance_calculator\n service_name: start_monitoring0\n play_messages_service_future_timeout_sec: 15.0\n playback_message_buffer_size: 100\n publish_tf_messages_in_set_data: false\n publish_tf_static_messages_in_set_data: false\n publisher_lower_frequency: 10.0\n publisher_upper_frequency: 200.0\n record_data_timeline: false\n revise_timestamps_as_message_ids: false\n set_data_service_future_timeout_sec: 60.0\n setup_service_client_timeout_sec: 20.0\n start_monitoring_service_timeout_sec: 10\n start_recording_service_future_timeout_sec: 35.0\n start_recording_service_timeout_sec: 30\n test_iterations: 5\n" - } -} \ No newline at end of file diff --git a/results/image_transport_h264_encoder_node-agx_xavier.json b/results/image_transport_h264_encoder_node-agx_xavier.json deleted file mode 100644 index ccaf6f5..0000000 --- a/results/image_transport_h264_encoder_node-agx_xavier.json +++ /dev/null @@ -1,51 +0,0 @@ -{ - "BasicPerformanceMetrics.RECEIVED_DURATION": 6106.235514322917, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 10.203322493182803, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 7.701981435850951, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 3.0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 50.0, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 135.46443684895834, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 1345.0929361979167, - "BasicPerformanceMetrics.MAX_JITTER": 4.422607421875, - "BasicPerformanceMetrics.MIN_JITTER": 0.012451171875, - "BasicPerformanceMetrics.MEAN_JITTER": 1.122834578804348, - "BasicPerformanceMetrics.STD_DEV_JITTER": 0.8867431300952578, - "CPUProfilingMetrics.MAX_CPU_UTIL": 16.0875, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.8625, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 9.941666666666668, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 6.289534387535472, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 15.170833333333334, - "10.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 6086.43994140625, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 10.204426857752532, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 7.722083919740579, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 3, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 50, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 131.422607421875, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 1318.0283203125, - "BasicPerformanceMetrics.MAX_JITTER": 6.751708984375, - "BasicPerformanceMetrics.MIN_JITTER": 0.006103515625, - "BasicPerformanceMetrics.MEAN_JITTER": 1.045041232638889, - "BasicPerformanceMetrics.STD_DEV_JITTER": 1.2384883121821242, - "CPUProfilingMetrics.MAX_CPU_UTIL": 16.45, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.625, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 9.627500000000001, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 6.623777717435874, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 15.6125 - }, - "metadata": { - "BenchmarkMetadata.NAME": "image_transport H264 Encoder Node Benchmark", - "BenchmarkMetadata.TEST_FILE_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/scripts/image_transport_h264_encoder_node.py", - "BenchmarkMetadata.TEST_DATETIME": "2023-05-25T06:18:11Z", - "BenchmarkMetadata.DEVICE_HOSTNAME": "ros-iyxkxzpofvhj3lap6mawpvnmja", - "BenchmarkMetadata.DEVICE_ARCH": "aarch64", - "BenchmarkMetadata.DEVICE_OS": "Linux 5.10.104-tegra #1 SMP PREEMPT Sun Mar 19 07:55:28 PDT 2023", - "BenchmarkMetadata.IDLE_CPU_UTIL": 2.0, - "BenchmarkMetadata.BENCHMARK_MODE": 1, - "BenchmarkMetadata.PEAK_THROUGHPUT_PREDICTION": 10.0, - "BenchmarkMetadata.INPUT_DATA_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets/datasets/r2b_dataset/r2b_mezzanine", - "BenchmarkMetadata.INPUT_DATA_SIZE": 2057437184, - "BenchmarkMetadata.INPUT_DATA_HASH": "c84d7420c5524d882f1019d350232fb1", - "BenchmarkMetadata.CONFIG": "ros2_benchmark_config:\n additional_fixed_publisher_rate_tests:\n - 10.0\n - 30.0\n - 60.0\n assets_root: /workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets\n benchmark_duration: 5.0\n benchmark_mode: BenchmarkMode.LOOPING\n benchmark_name: image_transport H264 Encoder Node Benchmark\n benchmark_namespace: r2b\n binary_search_acceptable_frame_loss_fraction: 0.05\n binary_search_acceptable_frame_rate_drop: 10.0\n binary_search_duration_fraction: 1.0\n binary_search_terminal_interval_width: 10.0\n cpu_profiling_interval_sec: 1.0\n custom_report_info: {}\n default_config_version: 0.30.0\n default_service_future_timeout_sec: 15.0\n enable_cpu_profiler: true\n enable_trial_buffer_preparation: false\n enforce_publisher_rate: false\n input_data_end_time: -1.0\n input_data_path: datasets/r2b_dataset/r2b_mezzanine\n input_data_start_time: -1.0\n linear_scan_acceptable_frame_loss_fraction: 0.05\n linear_scan_acceptable_frame_rate_drop: 5.0\n linear_scan_duration_fraction: 1.0\n linear_scan_step_size: 5.0\n load_data_in_real_time: false\n log_file_name: kpi.json\n log_folder: /osmo/data/output\n monitor_info_list:\n - calculators:\n - class_name: BasicPerformanceCalculator\n config:\n report_prefix: ''\n module_name: ros2_benchmark.basic_performance_calculator\n service_name: start_monitoring0\n play_messages_service_future_timeout_sec: 15.0\n playback_message_buffer_size: 100\n publish_tf_messages_in_set_data: false\n publish_tf_static_messages_in_set_data: false\n publisher_lower_frequency: 10.0\n publisher_upper_frequency: 200.0\n record_data_timeline: false\n revise_timestamps_as_message_ids: false\n set_data_service_future_timeout_sec: 60.0\n setup_service_client_timeout_sec: 20.0\n start_monitoring_service_timeout_sec: 10\n start_recording_service_future_timeout_sec: 35.0\n start_recording_service_timeout_sec: 30\n test_iterations: 5\n" - } -} \ No newline at end of file diff --git a/results/image_transport_h264_encoder_node-nuc_3060ti.json b/results/image_transport_h264_encoder_node-nuc_3060ti.json deleted file mode 100644 index 5ada0a3..0000000 --- a/results/image_transport_h264_encoder_node-nuc_3060ti.json +++ /dev/null @@ -1,69 +0,0 @@ -{ - "BasicPerformanceMetrics.RECEIVED_DURATION": 5190.85205078125, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 45.82608580246451, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 43.53815847469652, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 2.0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 228.0, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 23.731770833333332, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 239.41943359375, - "BasicPerformanceMetrics.MAX_JITTER": 8.08154296875, - "BasicPerformanceMetrics.MIN_JITTER": 0.000732421875, - "BasicPerformanceMetrics.MEAN_JITTER": 0.2629296439034598, - "BasicPerformanceMetrics.STD_DEV_JITTER": 0.6555536331300521, - "CPUProfilingMetrics.MAX_CPU_UTIL": 7.5625, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.1875, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 3.651515151515152, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 3.301931317785003, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 7.108333333333333, - "10.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4910.533447265625, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 10.202722313483227, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 10.182193143973379, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 50, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 23.523193359375, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 33.40380859375, - "BasicPerformanceMetrics.MAX_JITTER": 23.57177734375, - "BasicPerformanceMetrics.MIN_JITTER": 0.031494140625, - "BasicPerformanceMetrics.MEAN_JITTER": 5.2427012125651045, - "BasicPerformanceMetrics.STD_DEV_JITTER": 6.355367588057853, - "CPUProfilingMetrics.MAX_CPU_UTIL": 8.55, - "CPUProfilingMetrics.MIN_CPU_UTIL": 3.15625, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 6.9325, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 1.9305172558151353, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 7.30625 - }, - "30.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4978.1376953125, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 30.20097361765046, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 30.131749899413705, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 150, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 23.722412109375, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 35.1328125, - "BasicPerformanceMetrics.MAX_JITTER": 22.9345703125, - "BasicPerformanceMetrics.MIN_JITTER": 0.013427734375, - "BasicPerformanceMetrics.MEAN_JITTER": 2.9619140625, - "BasicPerformanceMetrics.STD_DEV_JITTER": 5.872157916983659, - "CPUProfilingMetrics.MAX_CPU_UTIL": 5.1875, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.375, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 4.202083333333333, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 1.7152755285635275, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 4.875 - }, - "metadata": { - "BenchmarkMetadata.NAME": "image_transport H264 Encoder Node Benchmark", - "BenchmarkMetadata.TEST_FILE_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/scripts/image_transport_h264_encoder_node.py", - "BenchmarkMetadata.TEST_DATETIME": "2023-05-25T07:07:42Z", - "BenchmarkMetadata.DEVICE_HOSTNAME": "ros-jypudj5rsbfmpo7lbdonftcgbm", - "BenchmarkMetadata.DEVICE_ARCH": "x86_64", - "BenchmarkMetadata.DEVICE_OS": "Linux 5.15.0-50-generic #56~20.04.1-Ubuntu SMP Tue Sep 27 15:51:29 UTC 2022", - "BenchmarkMetadata.IDLE_CPU_UTIL": 0.6875, - "BenchmarkMetadata.BENCHMARK_MODE": 1, - "BenchmarkMetadata.PEAK_THROUGHPUT_PREDICTION": 45.625, - "BenchmarkMetadata.INPUT_DATA_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets/datasets/r2b_dataset/r2b_mezzanine", - "BenchmarkMetadata.INPUT_DATA_SIZE": 2057437184, - "BenchmarkMetadata.INPUT_DATA_HASH": "c84d7420c5524d882f1019d350232fb1", - "BenchmarkMetadata.CONFIG": "ros2_benchmark_config:\n additional_fixed_publisher_rate_tests:\n - 10.0\n - 30.0\n - 60.0\n assets_root: /workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets\n benchmark_duration: 5.0\n benchmark_mode: BenchmarkMode.LOOPING\n benchmark_name: image_transport H264 Encoder Node Benchmark\n benchmark_namespace: r2b\n binary_search_acceptable_frame_loss_fraction: 0.05\n binary_search_acceptable_frame_rate_drop: 10.0\n binary_search_duration_fraction: 1.0\n binary_search_terminal_interval_width: 10.0\n cpu_profiling_interval_sec: 1.0\n custom_report_info: {}\n default_config_version: 0.30.0\n default_service_future_timeout_sec: 15.0\n enable_cpu_profiler: true\n enable_trial_buffer_preparation: false\n enforce_publisher_rate: false\n input_data_end_time: -1.0\n input_data_path: datasets/r2b_dataset/r2b_mezzanine\n input_data_start_time: -1.0\n linear_scan_acceptable_frame_loss_fraction: 0.05\n linear_scan_acceptable_frame_rate_drop: 5.0\n linear_scan_duration_fraction: 1.0\n linear_scan_step_size: 5.0\n load_data_in_real_time: false\n log_file_name: kpi.json\n log_folder: /osmo/data/output\n monitor_info_list:\n - calculators:\n - class_name: BasicPerformanceCalculator\n config:\n report_prefix: ''\n module_name: ros2_benchmark.basic_performance_calculator\n service_name: start_monitoring0\n play_messages_service_future_timeout_sec: 15.0\n playback_message_buffer_size: 100\n publish_tf_messages_in_set_data: false\n publish_tf_static_messages_in_set_data: false\n publisher_lower_frequency: 10.0\n publisher_upper_frequency: 200.0\n record_data_timeline: false\n revise_timestamps_as_message_ids: false\n set_data_service_future_timeout_sec: 60.0\n setup_service_client_timeout_sec: 20.0\n start_monitoring_service_timeout_sec: 10\n start_recording_service_future_timeout_sec: 35.0\n start_recording_service_timeout_sec: 30\n test_iterations: 5\n" - } -} \ No newline at end of file diff --git a/results/image_transport_h264_encoder_node-nuc_4060ti.json b/results/image_transport_h264_encoder_node-nuc_4060ti.json deleted file mode 100644 index 40fe398..0000000 --- a/results/image_transport_h264_encoder_node-nuc_4060ti.json +++ /dev/null @@ -1,69 +0,0 @@ -{ - "BasicPerformanceMetrics.RECEIVED_DURATION": 5188.654622395833, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 45.82574924713395, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 43.37938827539679, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 3.0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 228.0, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 23.861002604166668, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 237.13321940104166, - "BasicPerformanceMetrics.MAX_JITTER": 8.274658203125, - "BasicPerformanceMetrics.MIN_JITTER": 0.000244140625, - "BasicPerformanceMetrics.MEAN_JITTER": 0.25087978603603606, - "BasicPerformanceMetrics.STD_DEV_JITTER": 0.5722320975675211, - "CPUProfilingMetrics.MAX_CPU_UTIL": 7.43125, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.125, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 3.5736742424242425, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 3.289064221332725, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 7.247916666666666, - "10.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4923.89990234375, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 10.202875307761413, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 10.154552487186075, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 50, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 23.668701171875, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 46.9892578125, - "BasicPerformanceMetrics.MAX_JITTER": 31.049072265625, - "BasicPerformanceMetrics.MIN_JITTER": 0.12744140625, - "BasicPerformanceMetrics.MEAN_JITTER": 10.866973876953125, - "BasicPerformanceMetrics.STD_DEV_JITTER": 9.168423180007128, - "CPUProfilingMetrics.MAX_CPU_UTIL": 3.075, - "CPUProfilingMetrics.MIN_CPU_UTIL": 2.50625, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 2.7075, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 0.2018043607061057, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 2.68125 - }, - "30.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4975.437255859375, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 30.20139671628629, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 30.148104033138186, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 150, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 23.810546875, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 32.590087890625, - "BasicPerformanceMetrics.MAX_JITTER": 40.9716796875, - "BasicPerformanceMetrics.MIN_JITTER": 0.0, - "BasicPerformanceMetrics.MEAN_JITTER": 4.78186694995777, - "BasicPerformanceMetrics.STD_DEV_JITTER": 7.525075017838559, - "CPUProfilingMetrics.MAX_CPU_UTIL": 5.61875, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.0, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 4.440625, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 1.9912397663997339, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 5.36875 - }, - "metadata": { - "BenchmarkMetadata.NAME": "image_transport H264 Encoder Node Benchmark", - "BenchmarkMetadata.TEST_FILE_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/scripts/image_transport_h264_encoder_node.py", - "BenchmarkMetadata.TEST_DATETIME": "2023-05-25T06:16:35Z", - "BenchmarkMetadata.DEVICE_HOSTNAME": "ros-hogv5mka6nagbbp2yxemkhxq3a", - "BenchmarkMetadata.DEVICE_ARCH": "x86_64", - "BenchmarkMetadata.DEVICE_OS": "Linux 5.15.0-72-generic #79~20.04.1-Ubuntu SMP Thu Apr 20 22:12:07 UTC 2023", - "BenchmarkMetadata.IDLE_CPU_UTIL": 0.49375, - "BenchmarkMetadata.BENCHMARK_MODE": 1, - "BenchmarkMetadata.PEAK_THROUGHPUT_PREDICTION": 45.625, - "BenchmarkMetadata.INPUT_DATA_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets/datasets/r2b_dataset/r2b_mezzanine", - "BenchmarkMetadata.INPUT_DATA_SIZE": 2057437184, - "BenchmarkMetadata.INPUT_DATA_HASH": "c84d7420c5524d882f1019d350232fb1", - "BenchmarkMetadata.CONFIG": "ros2_benchmark_config:\n additional_fixed_publisher_rate_tests:\n - 10.0\n - 30.0\n - 60.0\n assets_root: /workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets\n benchmark_duration: 5.0\n benchmark_mode: BenchmarkMode.LOOPING\n benchmark_name: image_transport H264 Encoder Node Benchmark\n benchmark_namespace: r2b\n binary_search_acceptable_frame_loss_fraction: 0.05\n binary_search_acceptable_frame_rate_drop: 10.0\n binary_search_duration_fraction: 1.0\n binary_search_terminal_interval_width: 10.0\n cpu_profiling_interval_sec: 1.0\n custom_report_info: {}\n default_config_version: 0.30.0\n default_service_future_timeout_sec: 15.0\n enable_cpu_profiler: true\n enable_trial_buffer_preparation: false\n enforce_publisher_rate: false\n input_data_end_time: -1.0\n input_data_path: datasets/r2b_dataset/r2b_mezzanine\n input_data_start_time: -1.0\n linear_scan_acceptable_frame_loss_fraction: 0.05\n linear_scan_acceptable_frame_rate_drop: 5.0\n linear_scan_duration_fraction: 1.0\n linear_scan_step_size: 5.0\n load_data_in_real_time: false\n log_file_name: kpi.json\n log_folder: /osmo/data/output\n monitor_info_list:\n - calculators:\n - class_name: BasicPerformanceCalculator\n config:\n report_prefix: ''\n module_name: ros2_benchmark.basic_performance_calculator\n service_name: start_monitoring0\n play_messages_service_future_timeout_sec: 15.0\n playback_message_buffer_size: 100\n publish_tf_messages_in_set_data: false\n publish_tf_static_messages_in_set_data: false\n publisher_lower_frequency: 10.0\n publisher_upper_frequency: 200.0\n record_data_timeline: false\n revise_timestamps_as_message_ids: false\n set_data_service_future_timeout_sec: 60.0\n setup_service_client_timeout_sec: 20.0\n start_monitoring_service_timeout_sec: 10\n start_recording_service_future_timeout_sec: 35.0\n start_recording_service_timeout_sec: 30\n test_iterations: 5\n" - } -} \ No newline at end of file diff --git a/results/image_transport_h264_encoder_node-orin_nx.json b/results/image_transport_h264_encoder_node-orin_nx.json deleted file mode 100644 index c3bcfd9..0000000 --- a/results/image_transport_h264_encoder_node-orin_nx.json +++ /dev/null @@ -1,51 +0,0 @@ -{ - "BasicPerformanceMetrics.RECEIVED_DURATION": 5192.690755208333, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 10.204176370780585, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 9.628919330713579, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0.0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 50.0, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 107.8720703125, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 400.59375, - "BasicPerformanceMetrics.MAX_JITTER": 3.29736328125, - "BasicPerformanceMetrics.MIN_JITTER": 0.010009765625, - "BasicPerformanceMetrics.MEAN_JITTER": 0.4039052327473958, - "BasicPerformanceMetrics.STD_DEV_JITTER": 0.5084516664555763, - "CPUProfilingMetrics.MAX_CPU_UTIL": 15.5625, - "CPUProfilingMetrics.MIN_CPU_UTIL": 4.5875, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 12.4625, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 3.5101149029663983, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 13.745833333333332, - "10.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 5191.7509765625, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 10.204374996231905, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 9.630662222768127, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 50, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 105.22216796875, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 397.114013671875, - "BasicPerformanceMetrics.MAX_JITTER": 3.365234375, - "BasicPerformanceMetrics.MIN_JITTER": 0.006591796875, - "BasicPerformanceMetrics.MEAN_JITTER": 0.3003285725911458, - "BasicPerformanceMetrics.STD_DEV_JITTER": 0.4719388051021062, - "CPUProfilingMetrics.MAX_CPU_UTIL": 14.8625, - "CPUProfilingMetrics.MIN_CPU_UTIL": 4.4625, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 12.731250000000001, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 3.713388992887764, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 13.875 - }, - "metadata": { - "BenchmarkMetadata.NAME": "image_transport H264 Encoder Node Benchmark", - "BenchmarkMetadata.TEST_FILE_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/scripts/image_transport_h264_encoder_node.py", - "BenchmarkMetadata.TEST_DATETIME": "2023-05-25T06:20:07Z", - "BenchmarkMetadata.DEVICE_HOSTNAME": "ros-fj3dtqfpebfxndofd7ifapnsz4", - "BenchmarkMetadata.DEVICE_ARCH": "aarch64", - "BenchmarkMetadata.DEVICE_OS": "Linux 5.10.104-tegra #1 SMP PREEMPT Wed Mar 15 00:17:07 PDT 2023", - "BenchmarkMetadata.IDLE_CPU_UTIL": 0.125, - "BenchmarkMetadata.BENCHMARK_MODE": 1, - "BenchmarkMetadata.PEAK_THROUGHPUT_PREDICTION": 10.0, - "BenchmarkMetadata.INPUT_DATA_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets/datasets/r2b_dataset/r2b_mezzanine", - "BenchmarkMetadata.INPUT_DATA_SIZE": 2057437184, - "BenchmarkMetadata.INPUT_DATA_HASH": "c84d7420c5524d882f1019d350232fb1", - "BenchmarkMetadata.CONFIG": "ros2_benchmark_config:\n additional_fixed_publisher_rate_tests:\n - 10.0\n - 30.0\n - 60.0\n assets_root: /workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets\n benchmark_duration: 5.0\n benchmark_mode: BenchmarkMode.LOOPING\n benchmark_name: image_transport H264 Encoder Node Benchmark\n benchmark_namespace: r2b\n binary_search_acceptable_frame_loss_fraction: 0.05\n binary_search_acceptable_frame_rate_drop: 10.0\n binary_search_duration_fraction: 1.0\n binary_search_terminal_interval_width: 10.0\n cpu_profiling_interval_sec: 1.0\n custom_report_info: {}\n default_config_version: 0.30.0\n default_service_future_timeout_sec: 15.0\n enable_cpu_profiler: true\n enable_trial_buffer_preparation: false\n enforce_publisher_rate: false\n input_data_end_time: -1.0\n input_data_path: datasets/r2b_dataset/r2b_mezzanine\n input_data_start_time: -1.0\n linear_scan_acceptable_frame_loss_fraction: 0.05\n linear_scan_acceptable_frame_rate_drop: 5.0\n linear_scan_duration_fraction: 1.0\n linear_scan_step_size: 5.0\n load_data_in_real_time: false\n log_file_name: kpi.json\n log_folder: /osmo/data/output\n monitor_info_list:\n - calculators:\n - class_name: BasicPerformanceCalculator\n config:\n report_prefix: ''\n module_name: ros2_benchmark.basic_performance_calculator\n service_name: start_monitoring0\n play_messages_service_future_timeout_sec: 15.0\n playback_message_buffer_size: 100\n publish_tf_messages_in_set_data: false\n publish_tf_static_messages_in_set_data: false\n publisher_lower_frequency: 10.0\n publisher_upper_frequency: 200.0\n record_data_timeline: false\n revise_timestamps_as_message_ids: false\n set_data_service_future_timeout_sec: 60.0\n setup_service_client_timeout_sec: 20.0\n start_monitoring_service_timeout_sec: 10\n start_recording_service_future_timeout_sec: 35.0\n start_recording_service_timeout_sec: 30\n test_iterations: 5\n" - } -} \ No newline at end of file diff --git a/results/image_transport_h264_encoder_node-x86_4090.json b/results/image_transport_h264_encoder_node-x86_4090.json deleted file mode 100644 index ed98bb4..0000000 --- a/results/image_transport_h264_encoder_node-x86_4090.json +++ /dev/null @@ -1,69 +0,0 @@ -{ - "BasicPerformanceMetrics.RECEIVED_DURATION": 5175.490315755208, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 50.825950470799135, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 46.328984750720736, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 13.333333333333334, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 253.0, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 24.746500651041668, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 220.93660481770834, - "BasicPerformanceMetrics.MAX_JITTER": 2.42919921875, - "BasicPerformanceMetrics.MIN_JITTER": 0.00048828125, - "BasicPerformanceMetrics.MEAN_JITTER": 0.3013663633966245, - "BasicPerformanceMetrics.STD_DEV_JITTER": 0.3532897099491787, - "CPUProfilingMetrics.MAX_CPU_UTIL": 7.86875, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.125, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 3.671212121212122, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 3.426962835826155, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 7.541666666666667, - "10.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4925.310302734375, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 10.1949454774515, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 10.151644653178826, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 50, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 21.438232421875, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 42.357421875, - "BasicPerformanceMetrics.MAX_JITTER": 35.93115234375, - "BasicPerformanceMetrics.MIN_JITTER": 0.28515625, - "BasicPerformanceMetrics.MEAN_JITTER": 6.998148600260417, - "BasicPerformanceMetrics.STD_DEV_JITTER": 8.109318608654702, - "CPUProfilingMetrics.MAX_CPU_UTIL": 3.6125, - "CPUProfilingMetrics.MIN_CPU_UTIL": 3.20625, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 3.34625, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 0.15363308562936556, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 3.23125 - }, - "30.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4979.9775390625, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 30.20079992795851, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 30.12061777857698, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 150, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 22.2158203125, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 35.4375, - "BasicPerformanceMetrics.MAX_JITTER": 39.30126953125, - "BasicPerformanceMetrics.MIN_JITTER": 0.00048828125, - "BasicPerformanceMetrics.MEAN_JITTER": 12.31043140308277, - "BasicPerformanceMetrics.STD_DEV_JITTER": 8.64259080166581, - "CPUProfilingMetrics.MAX_CPU_UTIL": 5.29375, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.375, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 4.325, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 1.7717656859284001, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 5.168750000000001 - }, - "metadata": { - "BenchmarkMetadata.NAME": "image_transport H264 Encoder Node Benchmark", - "BenchmarkMetadata.TEST_FILE_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/scripts/image_transport_h264_encoder_node.py", - "BenchmarkMetadata.TEST_DATETIME": "2023-05-25T07:21:37Z", - "BenchmarkMetadata.DEVICE_HOSTNAME": "ros-mamag6avl5h4vpd5vcbjfqubm4", - "BenchmarkMetadata.DEVICE_ARCH": "x86_64", - "BenchmarkMetadata.DEVICE_OS": "Linux 5.4.0-131-generic #147-Ubuntu SMP Fri Oct 14 17:07:22 UTC 2022", - "BenchmarkMetadata.IDLE_CPU_UTIL": 0.75, - "BenchmarkMetadata.BENCHMARK_MODE": 1, - "BenchmarkMetadata.PEAK_THROUGHPUT_PREDICTION": 50.625, - "BenchmarkMetadata.INPUT_DATA_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets/datasets/r2b_dataset/r2b_mezzanine", - "BenchmarkMetadata.INPUT_DATA_SIZE": 2057437184, - "BenchmarkMetadata.INPUT_DATA_HASH": "c84d7420c5524d882f1019d350232fb1", - "BenchmarkMetadata.CONFIG": "ros2_benchmark_config:\n additional_fixed_publisher_rate_tests:\n - 10.0\n - 30.0\n - 60.0\n assets_root: /workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets\n benchmark_duration: 5.0\n benchmark_mode: BenchmarkMode.LOOPING\n benchmark_name: image_transport H264 Encoder Node Benchmark\n benchmark_namespace: r2b\n binary_search_acceptable_frame_loss_fraction: 0.05\n binary_search_acceptable_frame_rate_drop: 10.0\n binary_search_duration_fraction: 1.0\n binary_search_terminal_interval_width: 10.0\n cpu_profiling_interval_sec: 1.0\n custom_report_info: {}\n default_config_version: 0.30.0\n default_service_future_timeout_sec: 15.0\n enable_cpu_profiler: true\n enable_trial_buffer_preparation: false\n enforce_publisher_rate: false\n input_data_end_time: -1.0\n input_data_path: datasets/r2b_dataset/r2b_mezzanine\n input_data_start_time: -1.0\n linear_scan_acceptable_frame_loss_fraction: 0.05\n linear_scan_acceptable_frame_rate_drop: 5.0\n linear_scan_duration_fraction: 1.0\n linear_scan_step_size: 5.0\n load_data_in_real_time: false\n log_file_name: kpi.json\n log_folder: /osmo/data/output\n monitor_info_list:\n - calculators:\n - class_name: BasicPerformanceCalculator\n config:\n report_prefix: ''\n module_name: ros2_benchmark.basic_performance_calculator\n service_name: start_monitoring0\n play_messages_service_future_timeout_sec: 15.0\n playback_message_buffer_size: 100\n publish_tf_messages_in_set_data: false\n publish_tf_static_messages_in_set_data: false\n publisher_lower_frequency: 10.0\n publisher_upper_frequency: 200.0\n record_data_timeline: false\n revise_timestamps_as_message_ids: false\n set_data_service_future_timeout_sec: 60.0\n setup_service_client_timeout_sec: 20.0\n start_monitoring_service_timeout_sec: 10\n start_recording_service_future_timeout_sec: 35.0\n start_recording_service_timeout_sec: 30\n test_iterations: 5\n" - } -} \ No newline at end of file diff --git a/results/stereo_image_proc_graph-agx_orin.json b/results/stereo_image_proc_graph-agx_orin.json index bfd59e2..1a3dc9b 100644 --- a/results/stereo_image_proc_graph-agx_orin.json +++ b/results/stereo_image_proc_graph-agx_orin.json @@ -1,90 +1,111 @@ { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4989.249267578125, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 66.45046547852519, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 63.51344922927181, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 14.0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 331.0, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 31.77099609375, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 41.749430338541664, - "BasicPerformanceMetrics.MAX_JITTER": 203.43212890625, - "BasicPerformanceMetrics.MIN_JITTER": 0.00146484375, - "BasicPerformanceMetrics.MEAN_JITTER": 5.170474316072952, - "BasicPerformanceMetrics.STD_DEV_JITTER": 7.86658763214056, - "CPUProfilingMetrics.MAX_CPU_UTIL": 36.074999999999996, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.0, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 17.765000000000004, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 17.247510623648623, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 35.574999999999996, + "BasicPerformanceMetrics.RECEIVED_DURATION": 4986.668863932292, + "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 72.07541395504144, + "BasicPerformanceMetrics.MEAN_FRAME_RATE": 70.60115916282923, + "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 7.0, + "BasicPerformanceMetrics.NUM_FRAMES_SENT": 359.0, + "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 29.013916015625, + "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 34.939860026041664, + "BasicPerformanceMetrics.MAX_JITTER": 14.6669921875, + "BasicPerformanceMetrics.MIN_JITTER": 0.00244140625, + "BasicPerformanceMetrics.MEAN_JITTER": 1.4449937803944524, + "BasicPerformanceMetrics.STD_DEV_JITTER": 1.6425790715665194, + "ResourceMetrics.BASELINE_OVERALL_CPU_UTILIZATION": 30.972222222222218, + "ResourceMetrics.MAX_OVERALL_CPU_UTILIZATION": 56.916666666666664, + "ResourceMetrics.MIN_OVERALL_CPU_UTILIZATION": 12.5, + "ResourceMetrics.MEAN_OVERALL_CPU_UTILIZATION": 33.488320707070706, + "ResourceMetrics.STDDEV_OVERALL_CPU_UTILIZATION": 7.525930195127411, + "ResourceMetrics.BASELINE_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MAX_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MIN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_DEVICE_UTILIZATION": 0.0, "10.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4903.092041015625, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 10.203697794172783, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 10.197646624158216, + "BasicPerformanceMetrics.RECEIVED_DURATION": 4899.616943359375, + "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 10.20427839284409, + "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 27.2666015625, + "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 26.97802734375, "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, "BasicPerformanceMetrics.NUM_FRAMES_SENT": 50, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 28.589599609375, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 31.497314453125, - "BasicPerformanceMetrics.MAX_JITTER": 9.67138671875, - "BasicPerformanceMetrics.MIN_JITTER": 0.015380859375, - "BasicPerformanceMetrics.MEAN_JITTER": 2.9454193115234375, - "BasicPerformanceMetrics.STD_DEV_JITTER": 2.5912869518386934, - "CPUProfilingMetrics.MAX_CPU_UTIL": 6.8, - "CPUProfilingMetrics.MIN_CPU_UTIL": 5.658333333333334, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 6.251666666666667, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 0.4012965099385849, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 6.8 + "BasicPerformanceMetrics.MEAN_FRAME_RATE": 10.204879397310188, + "BasicPerformanceMetrics.MAX_JITTER": 4.256591796875, + "BasicPerformanceMetrics.MIN_JITTER": 0.0009765625, + "BasicPerformanceMetrics.MEAN_JITTER": 1.2327168782552083, + "BasicPerformanceMetrics.STD_DEV_JITTER": 1.1596704015427444, + "ResourceMetrics.BASELINE_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MAX_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MIN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.BASELINE_OVERALL_CPU_UTILIZATION": 8.333333333333334, + "ResourceMetrics.MEAN_OVERALL_CPU_UTILIZATION": 4.885195035460994, + "ResourceMetrics.STDDEV_OVERALL_CPU_UTILIZATION": 7.0649589230023535, + "ResourceMetrics.MAX_OVERALL_CPU_UTILIZATION": 33.25, + "ResourceMetrics.MIN_OVERALL_CPU_UTILIZATION": 0.0 }, "60.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4985.777099609375, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 60.19965533149811, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 60.1711616878148, + "BasicPerformanceMetrics.RECEIVED_DURATION": 4980.638671875, + "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 60.20074951108938, + "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 30.89697265625, + "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 28.208984375, "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, "BasicPerformanceMetrics.NUM_FRAMES_SENT": 300, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 28.9033203125, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 31.26318359375, - "BasicPerformanceMetrics.MAX_JITTER": 11.7705078125, - "BasicPerformanceMetrics.MIN_JITTER": 0.00439453125, - "BasicPerformanceMetrics.MEAN_JITTER": 1.148225310664849, - "BasicPerformanceMetrics.STD_DEV_JITTER": 1.9013500138038073, - "CPUProfilingMetrics.MAX_CPU_UTIL": 32.74166666666667, - "CPUProfilingMetrics.MIN_CPU_UTIL": 1.2333333333333334, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 26.947222222222223, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 11.509491520555041, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 32.325 + "BasicPerformanceMetrics.MEAN_FRAME_RATE": 60.23323910125419, + "BasicPerformanceMetrics.MAX_JITTER": 8.5546875, + "BasicPerformanceMetrics.MIN_JITTER": 0.002197265625, + "BasicPerformanceMetrics.MEAN_JITTER": 1.8041074611996644, + "BasicPerformanceMetrics.STD_DEV_JITTER": 1.5604508260414962, + "ResourceMetrics.BASELINE_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MAX_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MIN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.BASELINE_OVERALL_CPU_UTILIZATION": 29.166666666666668, + "ResourceMetrics.MEAN_OVERALL_CPU_UTILIZATION": 28.729640151515152, + "ResourceMetrics.STDDEV_OVERALL_CPU_UTILIZATION": 7.918232735646421, + "ResourceMetrics.MAX_OVERALL_CPU_UTILIZATION": 49.916666666666664, + "ResourceMetrics.MIN_OVERALL_CPU_UTILIZATION": 8.333333333333334 }, "30.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4969.768310546875, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 30.20162088913985, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 30.182493554411586, + "BasicPerformanceMetrics.RECEIVED_DURATION": 4967.61181640625, + "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 30.200655930326928, + "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 27.43701171875, + "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 28.269287109375, "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, "BasicPerformanceMetrics.NUM_FRAMES_SENT": 150, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 28.479248046875, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 31.626708984375, - "BasicPerformanceMetrics.MAX_JITTER": 10.4345703125, - "BasicPerformanceMetrics.MIN_JITTER": 0.0322265625, - "BasicPerformanceMetrics.MEAN_JITTER": 2.766979320629223, - "BasicPerformanceMetrics.STD_DEV_JITTER": 2.4175750853234845, - "CPUProfilingMetrics.MAX_CPU_UTIL": 15.858333333333334, - "CPUProfilingMetrics.MIN_CPU_UTIL": 14.566666666666668, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 15.028333333333332, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 0.4482248939489593, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 15.1 + "BasicPerformanceMetrics.MEAN_FRAME_RATE": 30.195596102055216, + "BasicPerformanceMetrics.MAX_JITTER": 5.586181640625, + "BasicPerformanceMetrics.MIN_JITTER": 0.01220703125, + "BasicPerformanceMetrics.MEAN_JITTER": 1.260433712521115, + "BasicPerformanceMetrics.STD_DEV_JITTER": 1.0193066631017405, + "ResourceMetrics.BASELINE_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MAX_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MIN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.BASELINE_OVERALL_CPU_UTILIZATION": 8.333333333333334, + "ResourceMetrics.MEAN_OVERALL_CPU_UTILIZATION": 12.987827715355806, + "ResourceMetrics.STDDEV_OVERALL_CPU_UTILIZATION": 7.29553322376145, + "ResourceMetrics.MAX_OVERALL_CPU_UTILIZATION": 36.083333333333336, + "ResourceMetrics.MIN_OVERALL_CPU_UTILIZATION": 0.0 }, "custom": { "data_resolution": "Quarter HD (960,540)" }, "metadata": { "BenchmarkMetadata.NAME": "Stereo Image Pointcloud Graph Benchmark", - "BenchmarkMetadata.TEST_FILE_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/scripts/stereo_image_proc_graph.py", - "BenchmarkMetadata.TEST_DATETIME": "2023-05-25T06:11:18Z", - "BenchmarkMetadata.DEVICE_HOSTNAME": "ros-56pvnwkqdje3lg7isekggisxqe", + "BenchmarkMetadata.TEST_FILE_PATH": "/opt/ros/humble/share/ros2_benchmark/scripts/stereo_image_proc_graph.py", + "BenchmarkMetadata.TEST_DATETIME": "2024-12-06T14:50:02Z", + "BenchmarkMetadata.DEVICE_HOSTNAME": "1329dfdc1bec4685-c902f1ddc0db4c3d", "BenchmarkMetadata.DEVICE_ARCH": "aarch64", - "BenchmarkMetadata.DEVICE_OS": "Linux 5.10.104-tegra #1 SMP PREEMPT Sun Mar 19 07:55:28 PDT 2023", - "BenchmarkMetadata.IDLE_CPU_UTIL": 1.0, + "BenchmarkMetadata.DEVICE_OS": "Linux 5.15.148-tegra #1 SMP PREEMPT Thu Sep 12 21:01:54 PDT 2024", + "BenchmarkMetadata.IDLE_CPU_UTIL": 0.1898148148148148, + "BenchmarkMetadata.IDLE_GPU_UTIL": 0.0, "BenchmarkMetadata.BENCHMARK_MODE": 1, - "BenchmarkMetadata.PEAK_THROUGHPUT_PREDICTION": 66.25, - "BenchmarkMetadata.INPUT_DATA_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets/datasets/r2b_dataset/r2b_hideaway", + "BenchmarkMetadata.PEAK_THROUGHPUT_PREDICTION": 71.875, + "BenchmarkMetadata.INPUT_DATA_PATH": "/workspaces/isaac_ros-dev/src/ros2_benchmark/assets/datasets/r2b_dataset/r2b_hideaway", "BenchmarkMetadata.INPUT_DATA_SIZE": 1913946112, "BenchmarkMetadata.INPUT_DATA_HASH": "96a3c2b8007173f9da57e848b4fcee8c", - "BenchmarkMetadata.CONFIG": "ros2_benchmark_config:\n additional_fixed_publisher_rate_tests:\n - 10.0\n - 30.0\n - 60.0\n assets_root: /workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets\n benchmark_duration: 5.0\n benchmark_mode: BenchmarkMode.LOOPING\n benchmark_name: Stereo Image Pointcloud Graph Benchmark\n benchmark_namespace: r2b\n binary_search_acceptable_frame_loss_fraction: 0.05\n binary_search_acceptable_frame_rate_drop: 10.0\n binary_search_duration_fraction: 1.0\n binary_search_terminal_interval_width: 10.0\n cpu_profiling_interval_sec: 1.0\n custom_report_info:\n data_resolution: Resolution(960, 540, Quarter HD)\n default_config_version: 0.30.0\n default_service_future_timeout_sec: 15.0\n enable_cpu_profiler: true\n enable_trial_buffer_preparation: false\n enforce_publisher_rate: false\n input_data_end_time: -1.0\n input_data_path: datasets/r2b_dataset/r2b_hideaway\n input_data_start_time: -1.0\n linear_scan_acceptable_frame_loss_fraction: 0.05\n linear_scan_acceptable_frame_rate_drop: 5.0\n linear_scan_duration_fraction: 1.0\n linear_scan_step_size: 5.0\n load_data_in_real_time: false\n log_file_name: kpi.json\n log_folder: /osmo/data/output\n monitor_info_list:\n - calculators:\n - class_name: BasicPerformanceCalculator\n config:\n report_prefix: ''\n module_name: ros2_benchmark.basic_performance_calculator\n service_name: start_monitoring0\n play_messages_service_future_timeout_sec: 15.0\n playback_message_buffer_size: 100\n publish_tf_messages_in_set_data: false\n publish_tf_static_messages_in_set_data: false\n publisher_lower_frequency: 10.0\n publisher_upper_frequency: 100.0\n record_data_timeline: false\n revise_timestamps_as_message_ids: false\n set_data_service_future_timeout_sec: 60.0\n setup_service_client_timeout_sec: 20.0\n start_monitoring_service_timeout_sec: 10\n start_recording_service_future_timeout_sec: 35.0\n start_recording_service_timeout_sec: 30\n test_iterations: 5\n" + "BenchmarkMetadata.CONFIG": "ros2_benchmark_config:\n additional_fixed_publisher_rate_tests:\n - 10.0\n - 30.0\n - 60.0\n assets_root: /workspaces/isaac_ros-dev/src/ros2_benchmark/assets\n benchmark_duration: 5.0\n benchmark_mode: BenchmarkMode.LOOPING\n benchmark_name: Stereo Image Pointcloud Graph Benchmark\n benchmark_namespace: r2b\n binary_search_acceptable_frame_loss_fraction: 0.05\n binary_search_acceptable_frame_rate_drop: 10.0\n binary_search_duration_fraction: 1.0\n binary_search_terminal_interval_width: 10.0\n collect_start_timestamps_from_monitors: false\n custom_report_info:\n data_resolution: Resolution(960, 540, Quarter HD)\n default_config_version: 0.30.0\n default_service_future_timeout_sec: 15.0\n enable_resource_profiler: true\n enable_trial_buffer_preparation: false\n enforce_publisher_rate: false\n export_monitor_raw_data: false\n input_data_end_time: -1.0\n input_data_path: datasets/r2b_dataset/r2b_hideaway\n input_data_start_time: -1.0\n linear_scan_acceptable_frame_loss_fraction: 0.05\n linear_scan_acceptable_frame_rate_drop: 5.0\n linear_scan_duration_fraction: 1.0\n linear_scan_step_size: 5.0\n load_data_in_real_time: false\n log_file_name: kpi.json\n log_folder: /osmo/data/output\n monitor_info_list:\n - calculators:\n - class_name: BasicPerformanceCalculator\n config:\n report_prefix: ''\n module_name: ros2_benchmark.basic_performance_calculator\n service_name: monitor_node0\n play_messages_service_future_timeout_sec: 15.0\n playback_message_buffer_size: 100\n post_trial_run_wait_time_sec: 0.0\n pre_stop_monitoring_wait_time_sec: 5.0\n pre_trial_run_wait_time_sec: 0.0\n publish_tf_messages_in_set_data: false\n publish_tf_static_messages_in_set_data: false\n publisher_lower_frequency: 10.0\n publisher_upper_frequency: 100.0\n record_data_timeline: false\n resource_profiling_interval_sec: 0.01\n revise_timestamps_as_message_ids: false\n ros1_ws: /workspaces/isaac_ros-dev/isaac_ros_1-dev\n set_data_service_future_timeout_sec: 60.0\n setup_service_client_timeout_sec: 20.0\n start_recording_service_future_timeout_sec: 35.0\n start_recording_service_timeout_sec: 30\n test_iterations: 5\n" } } \ No newline at end of file diff --git a/results/stereo_image_proc_graph-agx_xavier.json b/results/stereo_image_proc_graph-agx_xavier.json deleted file mode 100644 index 2cc4f61..0000000 --- a/results/stereo_image_proc_graph-agx_xavier.json +++ /dev/null @@ -1,54 +0,0 @@ -{ - "BasicPerformanceMetrics.RECEIVED_DURATION": 4931.998453776042, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 27.076272731229682, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 27.169610846004854, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0.0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 134.0, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 68.947998046875, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 47.323323567708336, - "BasicPerformanceMetrics.MAX_JITTER": 66.935302734375, - "BasicPerformanceMetrics.MIN_JITTER": 0.070068359375, - "BasicPerformanceMetrics.MEAN_JITTER": 19.007610899029356, - "BasicPerformanceMetrics.STD_DEV_JITTER": 13.11430052604778, - "CPUProfilingMetrics.MAX_CPU_UTIL": 35.2875, - "CPUProfilingMetrics.MIN_CPU_UTIL": 2.0, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 30.17097222222222, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 8.334481634056887, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 33.025, - "10.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4895.18798828125, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 10.203802012361148, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 10.214112332293801, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 50, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 60.6025390625, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 55.65625, - "BasicPerformanceMetrics.MAX_JITTER": 36.0625, - "BasicPerformanceMetrics.MIN_JITTER": 0.191650390625, - "BasicPerformanceMetrics.MEAN_JITTER": 12.643544514973959, - "BasicPerformanceMetrics.STD_DEV_JITTER": 8.991546494217538, - "CPUProfilingMetrics.MAX_CPU_UTIL": 13.85, - "CPUProfilingMetrics.MIN_CPU_UTIL": 12.25, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 13.084999999999999, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 0.646693513188434, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 12.675 - }, - "custom": { - "data_resolution": "Quarter HD (960,540)" - }, - "metadata": { - "BenchmarkMetadata.NAME": "Stereo Image Pointcloud Graph Benchmark", - "BenchmarkMetadata.TEST_FILE_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/scripts/stereo_image_proc_graph.py", - "BenchmarkMetadata.TEST_DATETIME": "2023-05-25T07:06:52Z", - "BenchmarkMetadata.DEVICE_HOSTNAME": "ros-xqvcwdkkcba7bhkpq3wqu7oeri", - "BenchmarkMetadata.DEVICE_ARCH": "aarch64", - "BenchmarkMetadata.DEVICE_OS": "Linux 5.10.104-tegra #1 SMP PREEMPT Sun Mar 19 07:55:28 PDT 2023", - "BenchmarkMetadata.IDLE_CPU_UTIL": 1.375, - "BenchmarkMetadata.BENCHMARK_MODE": 1, - "BenchmarkMetadata.PEAK_THROUGHPUT_PREDICTION": 26.875, - "BenchmarkMetadata.INPUT_DATA_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets/datasets/r2b_dataset/r2b_hideaway", - "BenchmarkMetadata.INPUT_DATA_SIZE": 1913946112, - "BenchmarkMetadata.INPUT_DATA_HASH": "96a3c2b8007173f9da57e848b4fcee8c", - "BenchmarkMetadata.CONFIG": "ros2_benchmark_config:\n additional_fixed_publisher_rate_tests:\n - 10.0\n - 30.0\n - 60.0\n assets_root: /workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets\n benchmark_duration: 5.0\n benchmark_mode: BenchmarkMode.LOOPING\n benchmark_name: Stereo Image Pointcloud Graph Benchmark\n benchmark_namespace: r2b\n binary_search_acceptable_frame_loss_fraction: 0.05\n binary_search_acceptable_frame_rate_drop: 10.0\n binary_search_duration_fraction: 1.0\n binary_search_terminal_interval_width: 10.0\n cpu_profiling_interval_sec: 1.0\n custom_report_info:\n data_resolution: Resolution(960, 540, Quarter HD)\n default_config_version: 0.30.0\n default_service_future_timeout_sec: 15.0\n enable_cpu_profiler: true\n enable_trial_buffer_preparation: false\n enforce_publisher_rate: false\n input_data_end_time: -1.0\n input_data_path: datasets/r2b_dataset/r2b_hideaway\n input_data_start_time: -1.0\n linear_scan_acceptable_frame_loss_fraction: 0.05\n linear_scan_acceptable_frame_rate_drop: 5.0\n linear_scan_duration_fraction: 1.0\n linear_scan_step_size: 5.0\n load_data_in_real_time: false\n log_file_name: kpi.json\n log_folder: /osmo/data/output\n monitor_info_list:\n - calculators:\n - class_name: BasicPerformanceCalculator\n config:\n report_prefix: ''\n module_name: ros2_benchmark.basic_performance_calculator\n service_name: start_monitoring0\n play_messages_service_future_timeout_sec: 15.0\n playback_message_buffer_size: 100\n publish_tf_messages_in_set_data: false\n publish_tf_static_messages_in_set_data: false\n publisher_lower_frequency: 10.0\n publisher_upper_frequency: 100.0\n record_data_timeline: false\n revise_timestamps_as_message_ids: false\n set_data_service_future_timeout_sec: 60.0\n setup_service_client_timeout_sec: 20.0\n start_monitoring_service_timeout_sec: 10\n start_recording_service_future_timeout_sec: 35.0\n start_recording_service_timeout_sec: 30\n test_iterations: 5\n" - } -} \ No newline at end of file diff --git a/results/stereo_image_proc_graph-nuc_3060ti.json b/results/stereo_image_proc_graph-nuc_3060ti.json deleted file mode 100644 index 91a5c3c..0000000 --- a/results/stereo_image_proc_graph-nuc_3060ti.json +++ /dev/null @@ -1,90 +0,0 @@ -{ - "BasicPerformanceMetrics.RECEIVED_DURATION": 4979.080322265625, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 99.57388502940591, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 99.61679325591575, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0.0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 496.0, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 18.453531901041668, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 16.198811848958332, - "BasicPerformanceMetrics.MAX_JITTER": 12.47607421875, - "BasicPerformanceMetrics.MIN_JITTER": 0.002197265625, - "BasicPerformanceMetrics.MEAN_JITTER": 2.2223562634425607, - "BasicPerformanceMetrics.STD_DEV_JITTER": 1.972311765126867, - "CPUProfilingMetrics.MAX_CPU_UTIL": 22.41875, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.25, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 17.974652777777777, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 7.903329355396212, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 21.743750000000002, - "10.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4919.8955078125, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 10.203182835057977, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 10.162817466469154, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 50, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 15.563232421875, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 35.027099609375, - "BasicPerformanceMetrics.MAX_JITTER": 28.684326171875, - "BasicPerformanceMetrics.MIN_JITTER": 0.08544921875, - "BasicPerformanceMetrics.MEAN_JITTER": 11.237462361653646, - "BasicPerformanceMetrics.STD_DEV_JITTER": 7.736589171743489, - "CPUProfilingMetrics.MAX_CPU_UTIL": 4.66875, - "CPUProfilingMetrics.MIN_CPU_UTIL": 3.875, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 4.206250000000001, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 0.28181886913405935, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 3.95 - }, - "60.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4985.725341796875, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 60.19849631132342, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 60.171786336685535, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 300, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 15.541015625, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 17.753173828125, - "BasicPerformanceMetrics.MAX_JITTER": 17.18017578125, - "BasicPerformanceMetrics.MIN_JITTER": 0.053466796875, - "BasicPerformanceMetrics.MEAN_JITTER": 5.1580761391044465, - "BasicPerformanceMetrics.STD_DEV_JITTER": 3.5472602487376377, - "CPUProfilingMetrics.MAX_CPU_UTIL": 15.45625, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.375, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 12.268749999999999, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 5.354556568163854, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 15.4375 - }, - "30.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4972.61328125, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 30.198757368319853, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 30.16522530830981, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 150, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 17.2900390625, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 22.8115234375, - "BasicPerformanceMetrics.MAX_JITTER": 37.29541015625, - "BasicPerformanceMetrics.MIN_JITTER": 0.098876953125, - "BasicPerformanceMetrics.MEAN_JITTER": 13.26849365234375, - "BasicPerformanceMetrics.STD_DEV_JITTER": 9.390680165712485, - "CPUProfilingMetrics.MAX_CPU_UTIL": 10.78125, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.1875, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 8.548958333333333, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 3.777881448357958, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 10.78125 - }, - "custom": { - "data_resolution": "Quarter HD (960,540)" - }, - "metadata": { - "BenchmarkMetadata.NAME": "Stereo Image Pointcloud Graph Benchmark", - "BenchmarkMetadata.TEST_FILE_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/scripts/stereo_image_proc_graph.py", - "BenchmarkMetadata.TEST_DATETIME": "2023-05-25T08:22:15Z", - "BenchmarkMetadata.DEVICE_HOSTNAME": "ros-33ehja6bzjb3tlxjmfeur6s5jm", - "BenchmarkMetadata.DEVICE_ARCH": "x86_64", - "BenchmarkMetadata.DEVICE_OS": "Linux 5.15.0-50-generic #56~20.04.1-Ubuntu SMP Tue Sep 27 15:51:29 UTC 2022", - "BenchmarkMetadata.IDLE_CPU_UTIL": 0.125, - "BenchmarkMetadata.BENCHMARK_MODE": 1, - "BenchmarkMetadata.PEAK_THROUGHPUT_PREDICTION": 99.375, - "BenchmarkMetadata.INPUT_DATA_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets/datasets/r2b_dataset/r2b_hideaway", - "BenchmarkMetadata.INPUT_DATA_SIZE": 1913946112, - "BenchmarkMetadata.INPUT_DATA_HASH": "96a3c2b8007173f9da57e848b4fcee8c", - "BenchmarkMetadata.CONFIG": "ros2_benchmark_config:\n additional_fixed_publisher_rate_tests:\n - 10.0\n - 30.0\n - 60.0\n assets_root: /workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets\n benchmark_duration: 5.0\n benchmark_mode: BenchmarkMode.LOOPING\n benchmark_name: Stereo Image Pointcloud Graph Benchmark\n benchmark_namespace: r2b\n binary_search_acceptable_frame_loss_fraction: 0.05\n binary_search_acceptable_frame_rate_drop: 10.0\n binary_search_duration_fraction: 1.0\n binary_search_terminal_interval_width: 10.0\n cpu_profiling_interval_sec: 1.0\n custom_report_info:\n data_resolution: Resolution(960, 540, Quarter HD)\n default_config_version: 0.30.0\n default_service_future_timeout_sec: 15.0\n enable_cpu_profiler: true\n enable_trial_buffer_preparation: false\n enforce_publisher_rate: false\n input_data_end_time: -1.0\n input_data_path: datasets/r2b_dataset/r2b_hideaway\n input_data_start_time: -1.0\n linear_scan_acceptable_frame_loss_fraction: 0.05\n linear_scan_acceptable_frame_rate_drop: 5.0\n linear_scan_duration_fraction: 1.0\n linear_scan_step_size: 5.0\n load_data_in_real_time: false\n log_file_name: kpi.json\n log_folder: /osmo/data/output\n monitor_info_list:\n - calculators:\n - class_name: BasicPerformanceCalculator\n config:\n report_prefix: ''\n module_name: ros2_benchmark.basic_performance_calculator\n service_name: start_monitoring0\n play_messages_service_future_timeout_sec: 15.0\n playback_message_buffer_size: 100\n publish_tf_messages_in_set_data: false\n publish_tf_static_messages_in_set_data: false\n publisher_lower_frequency: 10.0\n publisher_upper_frequency: 100.0\n record_data_timeline: false\n revise_timestamps_as_message_ids: false\n set_data_service_future_timeout_sec: 60.0\n setup_service_client_timeout_sec: 20.0\n start_monitoring_service_timeout_sec: 10\n start_recording_service_future_timeout_sec: 35.0\n start_recording_service_timeout_sec: 30\n test_iterations: 5\n" - } -} \ No newline at end of file diff --git a/results/stereo_image_proc_graph-nuc_4060ti.json b/results/stereo_image_proc_graph-nuc_4060ti.json deleted file mode 100644 index bf7d1c8..0000000 --- a/results/stereo_image_proc_graph-nuc_4060ti.json +++ /dev/null @@ -1,90 +0,0 @@ -{ - "BasicPerformanceMetrics.RECEIVED_DURATION": 4980.04736328125, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 99.57367519201439, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 99.40859703491277, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 1.0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 496.0, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 18.691813151041668, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 17.534423828125, - "BasicPerformanceMetrics.MAX_JITTER": 10.860595703125, - "BasicPerformanceMetrics.MIN_JITTER": 0.002197265625, - "BasicPerformanceMetrics.MEAN_JITTER": 2.336315124015498, - "BasicPerformanceMetrics.STD_DEV_JITTER": 2.0590849339826622, - "CPUProfilingMetrics.MAX_CPU_UTIL": 23.01875, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.0625, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 10.228787878787879, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 10.698247792962805, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 21.54791666666667, - "10.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4919.030029296875, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 10.203358718505973, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 10.164605562927816, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 50, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 16.29833984375, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 34.981201171875, - "BasicPerformanceMetrics.MAX_JITTER": 27.065185546875, - "BasicPerformanceMetrics.MIN_JITTER": 0.16650390625, - "BasicPerformanceMetrics.MEAN_JITTER": 8.192738850911459, - "BasicPerformanceMetrics.STD_DEV_JITTER": 6.2030894697162955, - "CPUProfilingMetrics.MAX_CPU_UTIL": 5.1875, - "CPUProfilingMetrics.MIN_CPU_UTIL": 3.69375, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 4.255, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 0.5165601368669479, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 5.1875 - }, - "60.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4982.9697265625, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 60.19793008868711, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 60.20506173272598, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 300, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 19.322509765625, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 18.732177734375, - "BasicPerformanceMetrics.MAX_JITTER": 23.657470703125, - "BasicPerformanceMetrics.MIN_JITTER": 0.010498046875, - "BasicPerformanceMetrics.MEAN_JITTER": 4.971362632393037, - "BasicPerformanceMetrics.STD_DEV_JITTER": 4.085067309984986, - "CPUProfilingMetrics.MAX_CPU_UTIL": 14.71875, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.625, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 12.043750000000001, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 5.111002225591376, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 14.4375 - }, - "30.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4980.719482421875, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 30.198393714802023, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 30.116130918311125, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 150, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 16.15234375, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 29.72021484375, - "BasicPerformanceMetrics.MAX_JITTER": 37.803955078125, - "BasicPerformanceMetrics.MIN_JITTER": 0.259521484375, - "BasicPerformanceMetrics.MEAN_JITTER": 15.080873231630068, - "BasicPerformanceMetrics.STD_DEV_JITTER": 9.962878762948133, - "CPUProfilingMetrics.MAX_CPU_UTIL": 11.025, - "CPUProfilingMetrics.MIN_CPU_UTIL": 1.24375, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 9.120833333333332, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 3.5370723877790358, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 10.50625 - }, - "custom": { - "data_resolution": "Quarter HD (960,540)" - }, - "metadata": { - "BenchmarkMetadata.NAME": "Stereo Image Pointcloud Graph Benchmark", - "BenchmarkMetadata.TEST_FILE_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/scripts/stereo_image_proc_graph.py", - "BenchmarkMetadata.TEST_DATETIME": "2023-05-25T06:31:09Z", - "BenchmarkMetadata.DEVICE_HOSTNAME": "ros-vpix2xcd2ndg5nq2p3l4gjircu", - "BenchmarkMetadata.DEVICE_ARCH": "x86_64", - "BenchmarkMetadata.DEVICE_OS": "Linux 5.15.0-72-generic #79~20.04.1-Ubuntu SMP Thu Apr 20 22:12:07 UTC 2023", - "BenchmarkMetadata.IDLE_CPU_UTIL": 0.0625, - "BenchmarkMetadata.BENCHMARK_MODE": 1, - "BenchmarkMetadata.PEAK_THROUGHPUT_PREDICTION": 99.375, - "BenchmarkMetadata.INPUT_DATA_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets/datasets/r2b_dataset/r2b_hideaway", - "BenchmarkMetadata.INPUT_DATA_SIZE": 1913946112, - "BenchmarkMetadata.INPUT_DATA_HASH": "96a3c2b8007173f9da57e848b4fcee8c", - "BenchmarkMetadata.CONFIG": "ros2_benchmark_config:\n additional_fixed_publisher_rate_tests:\n - 10.0\n - 30.0\n - 60.0\n assets_root: /workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets\n benchmark_duration: 5.0\n benchmark_mode: BenchmarkMode.LOOPING\n benchmark_name: Stereo Image Pointcloud Graph Benchmark\n benchmark_namespace: r2b\n binary_search_acceptable_frame_loss_fraction: 0.05\n binary_search_acceptable_frame_rate_drop: 10.0\n binary_search_duration_fraction: 1.0\n binary_search_terminal_interval_width: 10.0\n cpu_profiling_interval_sec: 1.0\n custom_report_info:\n data_resolution: Resolution(960, 540, Quarter HD)\n default_config_version: 0.30.0\n default_service_future_timeout_sec: 15.0\n enable_cpu_profiler: true\n enable_trial_buffer_preparation: false\n enforce_publisher_rate: false\n input_data_end_time: -1.0\n input_data_path: datasets/r2b_dataset/r2b_hideaway\n input_data_start_time: -1.0\n linear_scan_acceptable_frame_loss_fraction: 0.05\n linear_scan_acceptable_frame_rate_drop: 5.0\n linear_scan_duration_fraction: 1.0\n linear_scan_step_size: 5.0\n load_data_in_real_time: false\n log_file_name: kpi.json\n log_folder: /osmo/data/output\n monitor_info_list:\n - calculators:\n - class_name: BasicPerformanceCalculator\n config:\n report_prefix: ''\n module_name: ros2_benchmark.basic_performance_calculator\n service_name: start_monitoring0\n play_messages_service_future_timeout_sec: 15.0\n playback_message_buffer_size: 100\n publish_tf_messages_in_set_data: false\n publish_tf_static_messages_in_set_data: false\n publisher_lower_frequency: 10.0\n publisher_upper_frequency: 100.0\n record_data_timeline: false\n revise_timestamps_as_message_ids: false\n set_data_service_future_timeout_sec: 60.0\n setup_service_client_timeout_sec: 20.0\n start_monitoring_service_timeout_sec: 10\n start_recording_service_future_timeout_sec: 35.0\n start_recording_service_timeout_sec: 30\n test_iterations: 5\n" - } -} \ No newline at end of file diff --git a/results/stereo_image_proc_graph-orin_nano.json b/results/stereo_image_proc_graph-orin_nano.json index 96f54dc..3d4b2f0 100644 --- a/results/stereo_image_proc_graph-orin_nano.json +++ b/results/stereo_image_proc_graph-orin_nano.json @@ -1,72 +1,88 @@ { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4991.3642578125, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 43.95130282072574, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 37.979208649151, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 28.333333333333332, + "BasicPerformanceMetrics.RECEIVED_DURATION": 4962.894612630208, + "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 43.95112903283117, + "BasicPerformanceMetrics.MEAN_FRAME_RATE": 43.333934797215214, + "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 3.0, "BasicPerformanceMetrics.NUM_FRAMES_SENT": 218.0, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 45.667399088541664, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 77.93806966145833, - "BasicPerformanceMetrics.MAX_JITTER": 277.634521484375, - "BasicPerformanceMetrics.MIN_JITTER": 0.006103515625, - "BasicPerformanceMetrics.MEAN_JITTER": 14.306524832589286, - "BasicPerformanceMetrics.STD_DEV_JITTER": 19.146871842008867, - "CPUProfilingMetrics.MAX_CPU_UTIL": 70.78333333333335, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.8333333333333334, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 33.7860101010101, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 32.990370608675654, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 69.02222222222223, + "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 44.972981770833336, + "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 46.893961588541664, + "BasicPerformanceMetrics.MAX_JITTER": 32.029541015625, + "BasicPerformanceMetrics.MIN_JITTER": 0.003662109375, + "BasicPerformanceMetrics.MEAN_JITTER": 5.128101276901533, + "BasicPerformanceMetrics.STD_DEV_JITTER": 4.4060093428494795, + "ResourceMetrics.BASELINE_OVERALL_CPU_UTILIZATION": 46.166666666666664, + "ResourceMetrics.MAX_OVERALL_CPU_UTILIZATION": 100.0, + "ResourceMetrics.MIN_OVERALL_CPU_UTILIZATION": 25.0, + "ResourceMetrics.MEAN_OVERALL_CPU_UTILIZATION": 65.20347222222222, + "ResourceMetrics.STDDEV_OVERALL_CPU_UTILIZATION": 13.788202510859756, + "ResourceMetrics.BASELINE_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MAX_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MIN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_DEVICE_UTILIZATION": 0.0, "10.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4904.339599609375, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 10.203654074005687, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 10.195052562017207, + "BasicPerformanceMetrics.RECEIVED_DURATION": 4903.037841796875, + "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 10.203942328992042, + "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 40.682861328125, + "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 43.65380859375, "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, "BasicPerformanceMetrics.NUM_FRAMES_SENT": 50, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 47.28662109375, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 51.4208984375, - "BasicPerformanceMetrics.MAX_JITTER": 15.3720703125, - "BasicPerformanceMetrics.MIN_JITTER": 0.17724609375, - "BasicPerformanceMetrics.MEAN_JITTER": 5.641469319661458, - "BasicPerformanceMetrics.STD_DEV_JITTER": 3.81696301949271, - "CPUProfilingMetrics.MAX_CPU_UTIL": 22.75, - "CPUProfilingMetrics.MIN_CPU_UTIL": 17.28333333333333, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 18.95, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 1.9521782705480561, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 17.783333333333335 + "BasicPerformanceMetrics.MEAN_FRAME_RATE": 10.197759351103825, + "BasicPerformanceMetrics.MAX_JITTER": 10.47509765625, + "BasicPerformanceMetrics.MIN_JITTER": 0.2978515625, + "BasicPerformanceMetrics.MEAN_JITTER": 3.461085001627604, + "BasicPerformanceMetrics.STD_DEV_JITTER": 2.5416203490742064, + "ResourceMetrics.BASELINE_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MAX_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MIN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.BASELINE_OVERALL_CPU_UTILIZATION": 33.333333333333336, + "ResourceMetrics.MEAN_OVERALL_CPU_UTILIZATION": 15.131944444444446, + "ResourceMetrics.STDDEV_OVERALL_CPU_UTILIZATION": 18.194873563641867, + "ResourceMetrics.MAX_OVERALL_CPU_UTILIZATION": 77.66666666666667, + "ResourceMetrics.MIN_OVERALL_CPU_UTILIZATION": 0.0 }, "30.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4966.787109375, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 30.20137890136782, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 30.20060991075484, + "BasicPerformanceMetrics.RECEIVED_DURATION": 4962.683837890625, + "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 30.201225990848698, + "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 45.34619140625, + "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 41.34423828125, "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, "BasicPerformanceMetrics.NUM_FRAMES_SENT": 150, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 48.69775390625, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 48.82421875, - "BasicPerformanceMetrics.MAX_JITTER": 19.071533203125, - "BasicPerformanceMetrics.MIN_JITTER": 0.019775390625, - "BasicPerformanceMetrics.MEAN_JITTER": 1.741065772804054, - "BasicPerformanceMetrics.STD_DEV_JITTER": 2.9940262106587605, - "CPUProfilingMetrics.MAX_CPU_UTIL": 51.349999999999994, - "CPUProfilingMetrics.MIN_CPU_UTIL": 2.0, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 41.975, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 17.891692542547144, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 49.550000000000004 + "BasicPerformanceMetrics.MEAN_FRAME_RATE": 30.225580532600905, + "BasicPerformanceMetrics.MAX_JITTER": 23.23974609375, + "BasicPerformanceMetrics.MIN_JITTER": 0.004638671875, + "BasicPerformanceMetrics.MEAN_JITTER": 3.4094898120777026, + "BasicPerformanceMetrics.STD_DEV_JITTER": 3.62437878211412, + "ResourceMetrics.BASELINE_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MAX_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MIN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.BASELINE_OVERALL_CPU_UTILIZATION": 11.0, + "ResourceMetrics.MEAN_OVERALL_CPU_UTILIZATION": 41.94583333333333, + "ResourceMetrics.STDDEV_OVERALL_CPU_UTILIZATION": 23.035039357345244, + "ResourceMetrics.MAX_OVERALL_CPU_UTILIZATION": 83.33333333333333, + "ResourceMetrics.MIN_OVERALL_CPU_UTILIZATION": 0.0 }, "custom": { "data_resolution": "Quarter HD (960,540)" }, "metadata": { "BenchmarkMetadata.NAME": "Stereo Image Pointcloud Graph Benchmark", - "BenchmarkMetadata.TEST_FILE_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/scripts/stereo_image_proc_graph.py", - "BenchmarkMetadata.TEST_DATETIME": "2023-05-25T06:30:39Z", - "BenchmarkMetadata.DEVICE_HOSTNAME": "ros-22sacyjkzbhhbarswgtrn4hsce", + "BenchmarkMetadata.TEST_FILE_PATH": "/opt/ros/humble/share/ros2_benchmark/scripts/stereo_image_proc_graph.py", + "BenchmarkMetadata.TEST_DATETIME": "2024-12-06T13:03:37Z", + "BenchmarkMetadata.DEVICE_HOSTNAME": "aa34c8e2907c41f7-78c55e73fefd4315", "BenchmarkMetadata.DEVICE_ARCH": "aarch64", - "BenchmarkMetadata.DEVICE_OS": "Linux 5.10.104-tegra #1 SMP PREEMPT Wed Feb 1 14:17:37 PST 2023", - "BenchmarkMetadata.IDLE_CPU_UTIL": 0.3333333333333333, + "BenchmarkMetadata.DEVICE_OS": "Linux 5.15.148-tegra #1 SMP PREEMPT Thu Sep 12 21:01:54 PDT 2024", + "BenchmarkMetadata.IDLE_CPU_UTIL": 1.0185185185185184, + "BenchmarkMetadata.IDLE_GPU_UTIL": 0.0, "BenchmarkMetadata.BENCHMARK_MODE": 1, "BenchmarkMetadata.PEAK_THROUGHPUT_PREDICTION": 43.75, - "BenchmarkMetadata.INPUT_DATA_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets/datasets/r2b_dataset/r2b_hideaway", + "BenchmarkMetadata.INPUT_DATA_PATH": "/workspaces/isaac_ros-dev/src/ros2_benchmark/assets/datasets/r2b_dataset/r2b_hideaway", "BenchmarkMetadata.INPUT_DATA_SIZE": 1913946112, "BenchmarkMetadata.INPUT_DATA_HASH": "96a3c2b8007173f9da57e848b4fcee8c", - "BenchmarkMetadata.CONFIG": "ros2_benchmark_config:\n additional_fixed_publisher_rate_tests:\n - 10.0\n - 30.0\n - 60.0\n assets_root: /workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets\n benchmark_duration: 5.0\n benchmark_mode: BenchmarkMode.LOOPING\n benchmark_name: Stereo Image Pointcloud Graph Benchmark\n benchmark_namespace: r2b\n binary_search_acceptable_frame_loss_fraction: 0.05\n binary_search_acceptable_frame_rate_drop: 10.0\n binary_search_duration_fraction: 1.0\n binary_search_terminal_interval_width: 10.0\n cpu_profiling_interval_sec: 1.0\n custom_report_info:\n data_resolution: Resolution(960, 540, Quarter HD)\n default_config_version: 0.30.0\n default_service_future_timeout_sec: 15.0\n enable_cpu_profiler: true\n enable_trial_buffer_preparation: false\n enforce_publisher_rate: false\n input_data_end_time: -1.0\n input_data_path: datasets/r2b_dataset/r2b_hideaway\n input_data_start_time: -1.0\n linear_scan_acceptable_frame_loss_fraction: 0.05\n linear_scan_acceptable_frame_rate_drop: 5.0\n linear_scan_duration_fraction: 1.0\n linear_scan_step_size: 5.0\n load_data_in_real_time: false\n log_file_name: kpi.json\n log_folder: /osmo/data/output\n monitor_info_list:\n - calculators:\n - class_name: BasicPerformanceCalculator\n config:\n report_prefix: ''\n module_name: ros2_benchmark.basic_performance_calculator\n service_name: start_monitoring0\n play_messages_service_future_timeout_sec: 15.0\n playback_message_buffer_size: 100\n publish_tf_messages_in_set_data: false\n publish_tf_static_messages_in_set_data: false\n publisher_lower_frequency: 10.0\n publisher_upper_frequency: 100.0\n record_data_timeline: false\n revise_timestamps_as_message_ids: false\n set_data_service_future_timeout_sec: 60.0\n setup_service_client_timeout_sec: 20.0\n start_monitoring_service_timeout_sec: 10\n start_recording_service_future_timeout_sec: 35.0\n start_recording_service_timeout_sec: 30\n test_iterations: 5\n" + "BenchmarkMetadata.CONFIG": "ros2_benchmark_config:\n additional_fixed_publisher_rate_tests:\n - 10.0\n - 30.0\n - 60.0\n assets_root: /workspaces/isaac_ros-dev/src/ros2_benchmark/assets\n benchmark_duration: 5.0\n benchmark_mode: BenchmarkMode.LOOPING\n benchmark_name: Stereo Image Pointcloud Graph Benchmark\n benchmark_namespace: r2b\n binary_search_acceptable_frame_loss_fraction: 0.05\n binary_search_acceptable_frame_rate_drop: 10.0\n binary_search_duration_fraction: 1.0\n binary_search_terminal_interval_width: 10.0\n collect_start_timestamps_from_monitors: false\n custom_report_info:\n data_resolution: Resolution(960, 540, Quarter HD)\n default_config_version: 0.30.0\n default_service_future_timeout_sec: 15.0\n enable_resource_profiler: true\n enable_trial_buffer_preparation: false\n enforce_publisher_rate: false\n export_monitor_raw_data: false\n input_data_end_time: -1.0\n input_data_path: datasets/r2b_dataset/r2b_hideaway\n input_data_start_time: -1.0\n linear_scan_acceptable_frame_loss_fraction: 0.05\n linear_scan_acceptable_frame_rate_drop: 5.0\n linear_scan_duration_fraction: 1.0\n linear_scan_step_size: 5.0\n load_data_in_real_time: false\n log_file_name: kpi.json\n log_folder: /osmo/data/output\n monitor_info_list:\n - calculators:\n - class_name: BasicPerformanceCalculator\n config:\n report_prefix: ''\n module_name: ros2_benchmark.basic_performance_calculator\n service_name: monitor_node0\n play_messages_service_future_timeout_sec: 15.0\n playback_message_buffer_size: 100\n post_trial_run_wait_time_sec: 0.0\n pre_stop_monitoring_wait_time_sec: 5.0\n pre_trial_run_wait_time_sec: 0.0\n publish_tf_messages_in_set_data: false\n publish_tf_static_messages_in_set_data: false\n publisher_lower_frequency: 10.0\n publisher_upper_frequency: 100.0\n record_data_timeline: false\n resource_profiling_interval_sec: 0.01\n revise_timestamps_as_message_ids: false\n ros1_ws: /workspaces/isaac_ros-dev/isaac_ros_1-dev\n set_data_service_future_timeout_sec: 60.0\n setup_service_client_timeout_sec: 20.0\n start_recording_service_future_timeout_sec: 35.0\n start_recording_service_timeout_sec: 30\n test_iterations: 5\n" } } \ No newline at end of file diff --git a/results/stereo_image_proc_graph-orin_nano_emul.json b/results/stereo_image_proc_graph-orin_nano_emul.json deleted file mode 100644 index 77cf0d8..0000000 --- a/results/stereo_image_proc_graph-orin_nano_emul.json +++ /dev/null @@ -1,72 +0,0 @@ -{ - "BasicPerformanceMetrics.RECEIVED_DURATION": 4959.345052083333, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 43.95074327687784, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 43.43518551731705, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 2.6666666666666665, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 218.0, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 45.822998046875, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 44.186116536458336, - "BasicPerformanceMetrics.MAX_JITTER": 32.91796875, - "BasicPerformanceMetrics.MIN_JITTER": 0.01220703125, - "BasicPerformanceMetrics.MEAN_JITTER": 6.845767012266355, - "BasicPerformanceMetrics.STD_DEV_JITTER": 5.873691139895624, - "CPUProfilingMetrics.MAX_CPU_UTIL": 67.48333333333333, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.3333333333333333, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 32.85611111111111, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 32.37363145287954, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 65.60555555555555, - "10.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4894.29736328125, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 10.204892618170339, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 10.215971014576613, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 50, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 46.900634765625, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 41.58740234375, - "BasicPerformanceMetrics.MAX_JITTER": 14.45654296875, - "BasicPerformanceMetrics.MIN_JITTER": 0.035888671875, - "BasicPerformanceMetrics.MEAN_JITTER": 4.8848012288411455, - "BasicPerformanceMetrics.STD_DEV_JITTER": 3.2732035713360235, - "CPUProfilingMetrics.MAX_CPU_UTIL": 18.366666666666667, - "CPUProfilingMetrics.MIN_CPU_UTIL": 14.316666666666668, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 16.259999999999998, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 1.3977203662472055, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 16.3 - }, - "30.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4957.285400390625, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 30.19887462989298, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 30.258495907494105, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 150, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 49.223876953125, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 39.436767578125, - "BasicPerformanceMetrics.MAX_JITTER": 20.850830078125, - "BasicPerformanceMetrics.MIN_JITTER": 0.04931640625, - "BasicPerformanceMetrics.MEAN_JITTER": 4.095265981313345, - "BasicPerformanceMetrics.STD_DEV_JITTER": 4.022907717506158, - "CPUProfilingMetrics.MAX_CPU_UTIL": 48.56666666666666, - "CPUProfilingMetrics.MIN_CPU_UTIL": 1.8333333333333333, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 38.05555555555556, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 16.286667083579687, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 45.88333333333333 - }, - "custom": { - "data_resolution": "Quarter HD (960,540)" - }, - "metadata": { - "BenchmarkMetadata.NAME": "Stereo Image Pointcloud Graph Benchmark", - "BenchmarkMetadata.TEST_FILE_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/scripts/stereo_image_proc_graph.py", - "BenchmarkMetadata.TEST_DATETIME": "2023-05-25T06:13:11Z", - "BenchmarkMetadata.DEVICE_HOSTNAME": "ros-574gg4xuvnh3fbgd4r2bdeuyva", - "BenchmarkMetadata.DEVICE_ARCH": "aarch64", - "BenchmarkMetadata.DEVICE_OS": "Linux 5.10.104-tegra #1 SMP PREEMPT Sun Mar 19 07:55:28 PDT 2023", - "BenchmarkMetadata.IDLE_CPU_UTIL": 0.6666666666666666, - "BenchmarkMetadata.BENCHMARK_MODE": 1, - "BenchmarkMetadata.PEAK_THROUGHPUT_PREDICTION": 43.75, - "BenchmarkMetadata.INPUT_DATA_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets/datasets/r2b_dataset/r2b_hideaway", - "BenchmarkMetadata.INPUT_DATA_SIZE": 1913946112, - "BenchmarkMetadata.INPUT_DATA_HASH": "96a3c2b8007173f9da57e848b4fcee8c", - "BenchmarkMetadata.CONFIG": "ros2_benchmark_config:\n additional_fixed_publisher_rate_tests:\n - 10.0\n - 30.0\n - 60.0\n assets_root: /workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets\n benchmark_duration: 5.0\n benchmark_mode: BenchmarkMode.LOOPING\n benchmark_name: Stereo Image Pointcloud Graph Benchmark\n benchmark_namespace: r2b\n binary_search_acceptable_frame_loss_fraction: 0.05\n binary_search_acceptable_frame_rate_drop: 10.0\n binary_search_duration_fraction: 1.0\n binary_search_terminal_interval_width: 10.0\n cpu_profiling_interval_sec: 1.0\n custom_report_info:\n data_resolution: Resolution(960, 540, Quarter HD)\n default_config_version: 0.30.0\n default_service_future_timeout_sec: 15.0\n enable_cpu_profiler: true\n enable_trial_buffer_preparation: false\n enforce_publisher_rate: false\n input_data_end_time: -1.0\n input_data_path: datasets/r2b_dataset/r2b_hideaway\n input_data_start_time: -1.0\n linear_scan_acceptable_frame_loss_fraction: 0.05\n linear_scan_acceptable_frame_rate_drop: 5.0\n linear_scan_duration_fraction: 1.0\n linear_scan_step_size: 5.0\n load_data_in_real_time: false\n log_file_name: kpi.json\n log_folder: /osmo/data/output\n monitor_info_list:\n - calculators:\n - class_name: BasicPerformanceCalculator\n config:\n report_prefix: ''\n module_name: ros2_benchmark.basic_performance_calculator\n service_name: start_monitoring0\n play_messages_service_future_timeout_sec: 15.0\n playback_message_buffer_size: 100\n publish_tf_messages_in_set_data: false\n publish_tf_static_messages_in_set_data: false\n publisher_lower_frequency: 10.0\n publisher_upper_frequency: 100.0\n record_data_timeline: false\n revise_timestamps_as_message_ids: false\n set_data_service_future_timeout_sec: 60.0\n setup_service_client_timeout_sec: 20.0\n start_monitoring_service_timeout_sec: 10\n start_recording_service_future_timeout_sec: 35.0\n start_recording_service_timeout_sec: 30\n test_iterations: 5\n" - } -} \ No newline at end of file diff --git a/results/stereo_image_proc_graph-orin_nx.json b/results/stereo_image_proc_graph-orin_nx.json index 650612a..390c708 100644 --- a/results/stereo_image_proc_graph-orin_nx.json +++ b/results/stereo_image_proc_graph-orin_nx.json @@ -1,72 +1,111 @@ { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4981.243245442708, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 55.19997051928468, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 55.207103023374174, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0.0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 275.0, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 33.401692708333336, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 32.673014322916664, - "BasicPerformanceMetrics.MAX_JITTER": 7.102783203125, - "BasicPerformanceMetrics.MIN_JITTER": 0.0009765625, - "BasicPerformanceMetrics.MEAN_JITTER": 1.646890381753663, - "BasicPerformanceMetrics.STD_DEV_JITTER": 1.270113013454085, - "CPUProfilingMetrics.MAX_CPU_UTIL": 49.375, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.375, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 40.456250000000004, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 17.877109989020695, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 48.17083333333333, + "BasicPerformanceMetrics.RECEIVED_DURATION": 4979.2958984375, + "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 60.8249543228562, + "BasicPerformanceMetrics.MEAN_FRAME_RATE": 60.778703698736365, + "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0.3333333333333333, + "BasicPerformanceMetrics.NUM_FRAMES_SENT": 303.0, + "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 33.322265625, + "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 30.681966145833332, + "BasicPerformanceMetrics.MAX_JITTER": 14.478515625, + "BasicPerformanceMetrics.MIN_JITTER": 0.003662109375, + "BasicPerformanceMetrics.MEAN_JITTER": 3.056316465039716, + "BasicPerformanceMetrics.STD_DEV_JITTER": 2.29675792698024, + "ResourceMetrics.BASELINE_OVERALL_CPU_UTILIZATION": 54.791666666666664, + "ResourceMetrics.MAX_OVERALL_CPU_UTILIZATION": 79.125, + "ResourceMetrics.MIN_OVERALL_CPU_UTILIZATION": 18.75, + "ResourceMetrics.MEAN_OVERALL_CPU_UTILIZATION": 48.635850694444436, + "ResourceMetrics.STDDEV_OVERALL_CPU_UTILIZATION": 10.152990278487072, + "ResourceMetrics.BASELINE_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MAX_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MIN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_DEVICE_UTILIZATION": 0.0, "10.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4900.913818359375, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 10.203847258969109, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 10.202178992149253, + "BasicPerformanceMetrics.RECEIVED_DURATION": 4900.69775390625, + "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 10.204826514212138, + "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 30.76708984375, + "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 31.822509765625, "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, "BasicPerformanceMetrics.NUM_FRAMES_SENT": 50, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 31.71142578125, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 32.5126953125, - "BasicPerformanceMetrics.MAX_JITTER": 4.5830078125, - "BasicPerformanceMetrics.MIN_JITTER": 0.064453125, - "BasicPerformanceMetrics.MEAN_JITTER": 1.3347371419270833, - "BasicPerformanceMetrics.STD_DEV_JITTER": 1.1152589640863777, - "CPUProfilingMetrics.MAX_CPU_UTIL": 9.8875, - "CPUProfilingMetrics.MIN_CPU_UTIL": 9.1125, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 9.477500000000001, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 0.26069618332457384, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 9.6 + "BasicPerformanceMetrics.MEAN_FRAME_RATE": 10.20262879100144, + "BasicPerformanceMetrics.MAX_JITTER": 7.5771484375, + "BasicPerformanceMetrics.MIN_JITTER": 0.164306640625, + "BasicPerformanceMetrics.MEAN_JITTER": 2.9007364908854165, + "BasicPerformanceMetrics.STD_DEV_JITTER": 1.9429280681575196, + "ResourceMetrics.BASELINE_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MAX_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MIN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.BASELINE_OVERALL_CPU_UTILIZATION": 12.5, + "ResourceMetrics.MEAN_OVERALL_CPU_UTILIZATION": 8.14423076923077, + "ResourceMetrics.STDDEV_OVERALL_CPU_UTILIZATION": 11.887111349491393, + "ResourceMetrics.MAX_OVERALL_CPU_UTILIZATION": 56.25, + "ResourceMetrics.MIN_OVERALL_CPU_UTILIZATION": 0.0 + }, + "60.0fps": { + "BasicPerformanceMetrics.RECEIVED_DURATION": 4980.18701171875, + "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 60.20014490558968, + "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 33.962158203125, + "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 30.7724609375, + "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, + "BasicPerformanceMetrics.NUM_FRAMES_SENT": 300, + "BasicPerformanceMetrics.MEAN_FRAME_RATE": 60.23870173832382, + "BasicPerformanceMetrics.MAX_JITTER": 8.68408203125, + "BasicPerformanceMetrics.MIN_JITTER": 0.00341796875, + "BasicPerformanceMetrics.MEAN_JITTER": 2.594592203229866, + "BasicPerformanceMetrics.STD_DEV_JITTER": 2.013650705428144, + "ResourceMetrics.BASELINE_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MAX_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MIN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.BASELINE_OVERALL_CPU_UTILIZATION": 45.75, + "ResourceMetrics.MEAN_OVERALL_CPU_UTILIZATION": 49.063151041666664, + "ResourceMetrics.STDDEV_OVERALL_CPU_UTILIZATION": 10.28194651101198, + "ResourceMetrics.MAX_OVERALL_CPU_UTILIZATION": 83.25, + "ResourceMetrics.MIN_OVERALL_CPU_UTILIZATION": 18.75 }, "30.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4965.48388671875, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 30.201303188198803, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 30.208536251866033, + "BasicPerformanceMetrics.RECEIVED_DURATION": 4968.475341796875, + "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 30.200654445822416, + "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 30.2001953125, + "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 31.895751953125, "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, "BasicPerformanceMetrics.NUM_FRAMES_SENT": 150, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 32.43603515625, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 31.246826171875, - "BasicPerformanceMetrics.MAX_JITTER": 6.246826171875, - "BasicPerformanceMetrics.MIN_JITTER": 0.02685546875, - "BasicPerformanceMetrics.MEAN_JITTER": 1.7763308963260136, - "BasicPerformanceMetrics.STD_DEV_JITTER": 1.3891084490229282, - "CPUProfilingMetrics.MAX_CPU_UTIL": 27.725, - "CPUProfilingMetrics.MIN_CPU_UTIL": 26.4375, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 27.375, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 0.4767730067862491, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 27.475 + "BasicPerformanceMetrics.MEAN_FRAME_RATE": 30.190348080856474, + "BasicPerformanceMetrics.MAX_JITTER": 8.57470703125, + "BasicPerformanceMetrics.MIN_JITTER": 0.0107421875, + "BasicPerformanceMetrics.MEAN_JITTER": 2.7605161924619934, + "BasicPerformanceMetrics.STD_DEV_JITTER": 2.0723569149305496, + "ResourceMetrics.BASELINE_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MAX_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MIN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.BASELINE_OVERALL_CPU_UTILIZATION": 12.5, + "ResourceMetrics.MEAN_OVERALL_CPU_UTILIZATION": 22.487113402061855, + "ResourceMetrics.STDDEV_OVERALL_CPU_UTILIZATION": 9.891990546741878, + "ResourceMetrics.MAX_OVERALL_CPU_UTILIZATION": 47.875, + "ResourceMetrics.MIN_OVERALL_CPU_UTILIZATION": 6.25 }, "custom": { "data_resolution": "Quarter HD (960,540)" }, "metadata": { "BenchmarkMetadata.NAME": "Stereo Image Pointcloud Graph Benchmark", - "BenchmarkMetadata.TEST_FILE_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/scripts/stereo_image_proc_graph.py", - "BenchmarkMetadata.TEST_DATETIME": "2023-05-25T07:28:12Z", - "BenchmarkMetadata.DEVICE_HOSTNAME": "ros-jz447cosbfe3jljfp4exuj4kly", + "BenchmarkMetadata.TEST_FILE_PATH": "/opt/ros/humble/share/ros2_benchmark/scripts/stereo_image_proc_graph.py", + "BenchmarkMetadata.TEST_DATETIME": "2024-12-06T14:05:49Z", + "BenchmarkMetadata.DEVICE_HOSTNAME": "d18c22ca16484f17-129ac836e7b34151", "BenchmarkMetadata.DEVICE_ARCH": "aarch64", - "BenchmarkMetadata.DEVICE_OS": "Linux 5.10.104-tegra #1 SMP PREEMPT Wed Mar 15 00:17:07 PDT 2023", - "BenchmarkMetadata.IDLE_CPU_UTIL": 1.2375, + "BenchmarkMetadata.DEVICE_OS": "Linux 5.15.148-tegra #1 SMP PREEMPT Thu Sep 12 21:01:54 PDT 2024", + "BenchmarkMetadata.IDLE_CPU_UTIL": 0.3055555555555556, + "BenchmarkMetadata.IDLE_GPU_UTIL": 0.0, "BenchmarkMetadata.BENCHMARK_MODE": 1, - "BenchmarkMetadata.PEAK_THROUGHPUT_PREDICTION": 55.0, - "BenchmarkMetadata.INPUT_DATA_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets/datasets/r2b_dataset/r2b_hideaway", + "BenchmarkMetadata.PEAK_THROUGHPUT_PREDICTION": 60.625, + "BenchmarkMetadata.INPUT_DATA_PATH": "/workspaces/isaac_ros-dev/src/ros2_benchmark/assets/datasets/r2b_dataset/r2b_hideaway", "BenchmarkMetadata.INPUT_DATA_SIZE": 1913946112, "BenchmarkMetadata.INPUT_DATA_HASH": "96a3c2b8007173f9da57e848b4fcee8c", - "BenchmarkMetadata.CONFIG": "ros2_benchmark_config:\n additional_fixed_publisher_rate_tests:\n - 10.0\n - 30.0\n - 60.0\n assets_root: /workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets\n benchmark_duration: 5.0\n benchmark_mode: BenchmarkMode.LOOPING\n benchmark_name: Stereo Image Pointcloud Graph Benchmark\n benchmark_namespace: r2b\n binary_search_acceptable_frame_loss_fraction: 0.05\n binary_search_acceptable_frame_rate_drop: 10.0\n binary_search_duration_fraction: 1.0\n binary_search_terminal_interval_width: 10.0\n cpu_profiling_interval_sec: 1.0\n custom_report_info:\n data_resolution: Resolution(960, 540, Quarter HD)\n default_config_version: 0.30.0\n default_service_future_timeout_sec: 15.0\n enable_cpu_profiler: true\n enable_trial_buffer_preparation: false\n enforce_publisher_rate: false\n input_data_end_time: -1.0\n input_data_path: datasets/r2b_dataset/r2b_hideaway\n input_data_start_time: -1.0\n linear_scan_acceptable_frame_loss_fraction: 0.05\n linear_scan_acceptable_frame_rate_drop: 5.0\n linear_scan_duration_fraction: 1.0\n linear_scan_step_size: 5.0\n load_data_in_real_time: false\n log_file_name: kpi.json\n log_folder: /osmo/data/output\n monitor_info_list:\n - calculators:\n - class_name: BasicPerformanceCalculator\n config:\n report_prefix: ''\n module_name: ros2_benchmark.basic_performance_calculator\n service_name: start_monitoring0\n play_messages_service_future_timeout_sec: 15.0\n playback_message_buffer_size: 100\n publish_tf_messages_in_set_data: false\n publish_tf_static_messages_in_set_data: false\n publisher_lower_frequency: 10.0\n publisher_upper_frequency: 100.0\n record_data_timeline: false\n revise_timestamps_as_message_ids: false\n set_data_service_future_timeout_sec: 60.0\n setup_service_client_timeout_sec: 20.0\n start_monitoring_service_timeout_sec: 10\n start_recording_service_future_timeout_sec: 35.0\n start_recording_service_timeout_sec: 30\n test_iterations: 5\n" + "BenchmarkMetadata.CONFIG": "ros2_benchmark_config:\n additional_fixed_publisher_rate_tests:\n - 10.0\n - 30.0\n - 60.0\n assets_root: /workspaces/isaac_ros-dev/src/ros2_benchmark/assets\n benchmark_duration: 5.0\n benchmark_mode: BenchmarkMode.LOOPING\n benchmark_name: Stereo Image Pointcloud Graph Benchmark\n benchmark_namespace: r2b\n binary_search_acceptable_frame_loss_fraction: 0.05\n binary_search_acceptable_frame_rate_drop: 10.0\n binary_search_duration_fraction: 1.0\n binary_search_terminal_interval_width: 10.0\n collect_start_timestamps_from_monitors: false\n custom_report_info:\n data_resolution: Resolution(960, 540, Quarter HD)\n default_config_version: 0.30.0\n default_service_future_timeout_sec: 15.0\n enable_resource_profiler: true\n enable_trial_buffer_preparation: false\n enforce_publisher_rate: false\n export_monitor_raw_data: false\n input_data_end_time: -1.0\n input_data_path: datasets/r2b_dataset/r2b_hideaway\n input_data_start_time: -1.0\n linear_scan_acceptable_frame_loss_fraction: 0.05\n linear_scan_acceptable_frame_rate_drop: 5.0\n linear_scan_duration_fraction: 1.0\n linear_scan_step_size: 5.0\n load_data_in_real_time: false\n log_file_name: kpi.json\n log_folder: /osmo/data/output\n monitor_info_list:\n - calculators:\n - class_name: BasicPerformanceCalculator\n config:\n report_prefix: ''\n module_name: ros2_benchmark.basic_performance_calculator\n service_name: monitor_node0\n play_messages_service_future_timeout_sec: 15.0\n playback_message_buffer_size: 100\n post_trial_run_wait_time_sec: 0.0\n pre_stop_monitoring_wait_time_sec: 5.0\n pre_trial_run_wait_time_sec: 0.0\n publish_tf_messages_in_set_data: false\n publish_tf_static_messages_in_set_data: false\n publisher_lower_frequency: 10.0\n publisher_upper_frequency: 100.0\n record_data_timeline: false\n resource_profiling_interval_sec: 0.01\n revise_timestamps_as_message_ids: false\n ros1_ws: /workspaces/isaac_ros-dev/isaac_ros_1-dev\n set_data_service_future_timeout_sec: 60.0\n setup_service_client_timeout_sec: 20.0\n start_recording_service_future_timeout_sec: 35.0\n start_recording_service_timeout_sec: 30\n test_iterations: 5\n" } } \ No newline at end of file diff --git a/results/stereo_image_proc_graph-x86-4090.json b/results/stereo_image_proc_graph-x86-4090.json new file mode 100644 index 0000000..d43610c --- /dev/null +++ b/results/stereo_image_proc_graph-x86-4090.json @@ -0,0 +1,131 @@ +{ + "BasicPerformanceMetrics.RECEIVED_DURATION": 4993.850504557292, + "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 94.57554681296796, + "BasicPerformanceMetrics.MEAN_FRAME_RATE": 91.1788726073172, + "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 15.333333333333334, + "BasicPerformanceMetrics.NUM_FRAMES_SENT": 471.0, + "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 18.952880859375, + "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 32.472005208333336, + "BasicPerformanceMetrics.MAX_JITTER": 19.553955078125, + "BasicPerformanceMetrics.MIN_JITTER": 0.002197265625, + "BasicPerformanceMetrics.MEAN_JITTER": 4.622789771155974, + "BasicPerformanceMetrics.STD_DEV_JITTER": 3.943583699602034, + "ResourceMetrics.BASELINE_OVERALL_CPU_UTILIZATION": 10.069791666666667, + "ResourceMetrics.MAX_OVERALL_CPU_UTILIZATION": 22.396875, + "ResourceMetrics.MIN_OVERALL_CPU_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_OVERALL_CPU_UTILIZATION": 4.041738859612242, + "ResourceMetrics.STDDEV_OVERALL_CPU_UTILIZATION": 4.847483374321279, + "ResourceMetrics.BASELINE_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MAX_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MIN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.BASELINE_DEVICE_MEMORY_UTILIZATION": 1.9400000000000002, + "ResourceMetrics.MAX_DEVICE_MEMORY_UTILIZATION": 1.94, + "ResourceMetrics.MIN_DEVICE_MEMORY_UTILIZATION": 1.94, + "ResourceMetrics.MEAN_DEVICE_MEMORY_UTILIZATION": 1.9400000000000004, + "ResourceMetrics.STDDEV_DEVICE_MEMORY_UTILIZATION": 3.7007434154171886e-16, + "10.0fps": { + "BasicPerformanceMetrics.RECEIVED_DURATION": 4903.20849609375, + "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 10.203657632609932, + "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 16.533447265625, + "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 19.538330078125, + "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, + "BasicPerformanceMetrics.NUM_FRAMES_SENT": 50, + "BasicPerformanceMetrics.MEAN_FRAME_RATE": 10.19740442198891, + "BasicPerformanceMetrics.MAX_JITTER": 15.119140625, + "BasicPerformanceMetrics.MIN_JITTER": 0.0498046875, + "BasicPerformanceMetrics.MEAN_JITTER": 4.9464874267578125, + "BasicPerformanceMetrics.STD_DEV_JITTER": 4.127654899117796, + "ResourceMetrics.BASELINE_OVERALL_CPU_UTILIZATION": 7.8125, + "ResourceMetrics.MEAN_OVERALL_CPU_UTILIZATION": 0.5062095032397408, + "ResourceMetrics.MAX_OVERALL_CPU_UTILIZATION": 9.375, + "ResourceMetrics.MIN_OVERALL_CPU_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_OVERALL_CPU_UTILIZATION": 1.542809051115343, + "ResourceMetrics.BASELINE_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MAX_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MIN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.BASELINE_DEVICE_MEMORY_UTILIZATION": 1.94, + "ResourceMetrics.MEAN_DEVICE_MEMORY_UTILIZATION": 1.9400000000000004, + "ResourceMetrics.STDDEV_DEVICE_MEMORY_UTILIZATION": 4.440892098500626e-16, + "ResourceMetrics.MAX_DEVICE_MEMORY_UTILIZATION": 1.94, + "ResourceMetrics.MIN_DEVICE_MEMORY_UTILIZATION": 1.94 + }, + "60.0fps": { + "BasicPerformanceMetrics.RECEIVED_DURATION": 4979.684814453125, + "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 60.1981896060732, + "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 22.88330078125, + "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 19.029541015625, + "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, + "BasicPerformanceMetrics.NUM_FRAMES_SENT": 300, + "BasicPerformanceMetrics.MEAN_FRAME_RATE": 60.244776763636665, + "BasicPerformanceMetrics.MAX_JITTER": 19.46484375, + "BasicPerformanceMetrics.MIN_JITTER": 0.0263671875, + "BasicPerformanceMetrics.MEAN_JITTER": 4.8843985947986575, + "BasicPerformanceMetrics.STD_DEV_JITTER": 3.8974152067312025, + "ResourceMetrics.BASELINE_OVERALL_CPU_UTILIZATION": 10.9375, + "ResourceMetrics.MEAN_OVERALL_CPU_UTILIZATION": 2.4515153556034486, + "ResourceMetrics.MAX_OVERALL_CPU_UTILIZATION": 17.1875, + "ResourceMetrics.MIN_OVERALL_CPU_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_OVERALL_CPU_UTILIZATION": 3.2728731777581155, + "ResourceMetrics.BASELINE_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MAX_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MIN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.BASELINE_DEVICE_MEMORY_UTILIZATION": 1.94, + "ResourceMetrics.MEAN_DEVICE_MEMORY_UTILIZATION": 1.9400000000000002, + "ResourceMetrics.STDDEV_DEVICE_MEMORY_UTILIZATION": 2.220446049250313e-16, + "ResourceMetrics.MAX_DEVICE_MEMORY_UTILIZATION": 1.94, + "ResourceMetrics.MIN_DEVICE_MEMORY_UTILIZATION": 1.94 + }, + "30.0fps": { + "BasicPerformanceMetrics.RECEIVED_DURATION": 4973.349853515625, + "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 30.200042842378483, + "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 16.2763671875, + "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 22.745849609375, + "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, + "BasicPerformanceMetrics.NUM_FRAMES_SENT": 150, + "BasicPerformanceMetrics.MEAN_FRAME_RATE": 30.160757722275676, + "BasicPerformanceMetrics.MAX_JITTER": 19.825439453125, + "BasicPerformanceMetrics.MIN_JITTER": 0.0283203125, + "BasicPerformanceMetrics.MEAN_JITTER": 5.190820642419763, + "BasicPerformanceMetrics.STD_DEV_JITTER": 4.367127920815086, + "ResourceMetrics.BASELINE_OVERALL_CPU_UTILIZATION": 4.6875, + "ResourceMetrics.MEAN_OVERALL_CPU_UTILIZATION": 1.271915236051502, + "ResourceMetrics.MAX_OVERALL_CPU_UTILIZATION": 14.0625, + "ResourceMetrics.MIN_OVERALL_CPU_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_OVERALL_CPU_UTILIZATION": 2.365817242570806, + "ResourceMetrics.BASELINE_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MAX_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MIN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.BASELINE_DEVICE_MEMORY_UTILIZATION": 1.94, + "ResourceMetrics.MEAN_DEVICE_MEMORY_UTILIZATION": 1.9400000000000002, + "ResourceMetrics.STDDEV_DEVICE_MEMORY_UTILIZATION": 2.220446049250313e-16, + "ResourceMetrics.MAX_DEVICE_MEMORY_UTILIZATION": 1.94, + "ResourceMetrics.MIN_DEVICE_MEMORY_UTILIZATION": 1.94 + }, + "custom": { + "data_resolution": "Quarter HD (960,540)" + }, + "metadata": { + "BenchmarkMetadata.NAME": "Stereo Image Pointcloud Graph Benchmark", + "BenchmarkMetadata.TEST_FILE_PATH": "/opt/ros/humble/share/ros2_benchmark/scripts/stereo_image_proc_graph.py", + "BenchmarkMetadata.TEST_DATETIME": "2024-12-06T21:19:06Z", + "BenchmarkMetadata.DEVICE_HOSTNAME": "0a37c1e79f424944-613de6877da14f3b", + "BenchmarkMetadata.DEVICE_ARCH": "x86_64", + "BenchmarkMetadata.DEVICE_OS": "Linux 6.8.0-48-generic #48~22.04.1-Ubuntu SMP PREEMPT_DYNAMIC Mon Oct 7 11:24:13 UTC 2", + "BenchmarkMetadata.IDLE_CPU_UTIL": 0.0, + "BenchmarkMetadata.IDLE_GPU_UTIL": 0, + "BenchmarkMetadata.BENCHMARK_MODE": 1, + "BenchmarkMetadata.PEAK_THROUGHPUT_PREDICTION": 94.375, + "BenchmarkMetadata.INPUT_DATA_PATH": "/workspaces/isaac_ros-dev/src/ros2_benchmark/assets/datasets/r2b_dataset/r2b_hideaway", + "BenchmarkMetadata.INPUT_DATA_SIZE": 1913946112, + "BenchmarkMetadata.INPUT_DATA_HASH": "96a3c2b8007173f9da57e848b4fcee8c", + "BenchmarkMetadata.CONFIG": "ros2_benchmark_config:\n additional_fixed_publisher_rate_tests:\n - 10.0\n - 30.0\n - 60.0\n assets_root: /workspaces/isaac_ros-dev/src/ros2_benchmark/assets\n benchmark_duration: 5.0\n benchmark_mode: BenchmarkMode.LOOPING\n benchmark_name: Stereo Image Pointcloud Graph Benchmark\n benchmark_namespace: r2b\n binary_search_acceptable_frame_loss_fraction: 0.05\n binary_search_acceptable_frame_rate_drop: 10.0\n binary_search_duration_fraction: 1.0\n binary_search_terminal_interval_width: 10.0\n collect_start_timestamps_from_monitors: false\n custom_report_info:\n data_resolution: Resolution(960, 540, Quarter HD)\n default_config_version: 0.30.0\n default_service_future_timeout_sec: 15.0\n enable_resource_profiler: true\n enable_trial_buffer_preparation: false\n enforce_publisher_rate: false\n export_monitor_raw_data: false\n input_data_end_time: -1.0\n input_data_path: datasets/r2b_dataset/r2b_hideaway\n input_data_start_time: -1.0\n linear_scan_acceptable_frame_loss_fraction: 0.05\n linear_scan_acceptable_frame_rate_drop: 5.0\n linear_scan_duration_fraction: 1.0\n linear_scan_step_size: 5.0\n load_data_in_real_time: false\n log_file_name: kpi.json\n log_folder: /osmo/data/output\n monitor_info_list:\n - calculators:\n - class_name: BasicPerformanceCalculator\n config:\n report_prefix: ''\n module_name: ros2_benchmark.basic_performance_calculator\n service_name: monitor_node0\n play_messages_service_future_timeout_sec: 15.0\n playback_message_buffer_size: 100\n post_trial_run_wait_time_sec: 0.0\n pre_stop_monitoring_wait_time_sec: 5.0\n pre_trial_run_wait_time_sec: 0.0\n publish_tf_messages_in_set_data: false\n publish_tf_static_messages_in_set_data: false\n publisher_lower_frequency: 10.0\n publisher_upper_frequency: 100.0\n record_data_timeline: false\n resource_profiling_interval_sec: 0.01\n revise_timestamps_as_message_ids: false\n ros1_ws: /workspaces/isaac_ros-dev/isaac_ros_1-dev\n set_data_service_future_timeout_sec: 60.0\n setup_service_client_timeout_sec: 20.0\n start_recording_service_future_timeout_sec: 35.0\n start_recording_service_timeout_sec: 30\n test_iterations: 5\n" + } +} \ No newline at end of file diff --git a/results/stereo_image_proc_graph-x86_4090.json b/results/stereo_image_proc_graph-x86_4090.json deleted file mode 100644 index f664464..0000000 --- a/results/stereo_image_proc_graph-x86_4090.json +++ /dev/null @@ -1,90 +0,0 @@ -{ - "BasicPerformanceMetrics.RECEIVED_DURATION": 4981.583658854167, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 99.57507421700252, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 99.56673173819665, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0.0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 496.0, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 16.372721354166668, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 16.4189453125, - "BasicPerformanceMetrics.MAX_JITTER": 11.748046875, - "BasicPerformanceMetrics.MIN_JITTER": 0.004150390625, - "BasicPerformanceMetrics.MEAN_JITTER": 2.4028404328504553, - "BasicPerformanceMetrics.STD_DEV_JITTER": 2.0436432407363454, - "CPUProfilingMetrics.MAX_CPU_UTIL": 22.51875, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.375, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 18.025347222222223, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 7.842786824078973, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 21.60833333333333, - "10.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4933.242431640625, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 10.202697407872208, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 10.135321888766722, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 50, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 14.78662109375, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 47.3642578125, - "BasicPerformanceMetrics.MAX_JITTER": 39.010986328125, - "BasicPerformanceMetrics.MIN_JITTER": 0.26953125, - "BasicPerformanceMetrics.MEAN_JITTER": 15.736104329427084, - "BasicPerformanceMetrics.STD_DEV_JITTER": 9.166923700394708, - "CPUProfilingMetrics.MAX_CPU_UTIL": 7.80625, - "CPUProfilingMetrics.MIN_CPU_UTIL": 5.425000000000001, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 6.723750000000001, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 0.7594344112561663, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 5.425000000000001 - }, - "60.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4980.0234375, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 60.199758554288685, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 60.24068034318363, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 300, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 19.2998046875, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 15.91455078125, - "BasicPerformanceMetrics.MAX_JITTER": 18.960693359375, - "BasicPerformanceMetrics.MIN_JITTER": 0.000244140625, - "BasicPerformanceMetrics.MEAN_JITTER": 4.221237285025167, - "BasicPerformanceMetrics.STD_DEV_JITTER": 3.5934907406017325, - "CPUProfilingMetrics.MAX_CPU_UTIL": 14.987499999999999, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.4375, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 12.172916666666666, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 5.2545251661993415, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 14.2 - }, - "30.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4964.37451171875, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 30.200317467074207, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 30.21528686965792, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 150, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 19.5126953125, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 17.052001953125, - "BasicPerformanceMetrics.MAX_JITTER": 42.37158203125, - "BasicPerformanceMetrics.MIN_JITTER": 0.053466796875, - "BasicPerformanceMetrics.MEAN_JITTER": 17.189830883129222, - "BasicPerformanceMetrics.STD_DEV_JITTER": 11.222895515511452, - "CPUProfilingMetrics.MAX_CPU_UTIL": 11.868750000000002, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.125, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 9.203125, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 4.096789186744298, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 11.868750000000002 - }, - "custom": { - "data_resolution": "Quarter HD (960,540)" - }, - "metadata": { - "BenchmarkMetadata.NAME": "Stereo Image Pointcloud Graph Benchmark", - "BenchmarkMetadata.TEST_FILE_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/scripts/stereo_image_proc_graph.py", - "BenchmarkMetadata.TEST_DATETIME": "2023-05-25T07:11:25Z", - "BenchmarkMetadata.DEVICE_HOSTNAME": "ros-a7b3fqfwrvftzmgd3kbrwu4lhm", - "BenchmarkMetadata.DEVICE_ARCH": "x86_64", - "BenchmarkMetadata.DEVICE_OS": "Linux 5.4.0-131-generic #147-Ubuntu SMP Fri Oct 14 17:07:22 UTC 2022", - "BenchmarkMetadata.IDLE_CPU_UTIL": 0.25, - "BenchmarkMetadata.BENCHMARK_MODE": 1, - "BenchmarkMetadata.PEAK_THROUGHPUT_PREDICTION": 99.375, - "BenchmarkMetadata.INPUT_DATA_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets/datasets/r2b_dataset/r2b_hideaway", - "BenchmarkMetadata.INPUT_DATA_SIZE": 1913946112, - "BenchmarkMetadata.INPUT_DATA_HASH": "96a3c2b8007173f9da57e848b4fcee8c", - "BenchmarkMetadata.CONFIG": "ros2_benchmark_config:\n additional_fixed_publisher_rate_tests:\n - 10.0\n - 30.0\n - 60.0\n assets_root: /workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets\n benchmark_duration: 5.0\n benchmark_mode: BenchmarkMode.LOOPING\n benchmark_name: Stereo Image Pointcloud Graph Benchmark\n benchmark_namespace: r2b\n binary_search_acceptable_frame_loss_fraction: 0.05\n binary_search_acceptable_frame_rate_drop: 10.0\n binary_search_duration_fraction: 1.0\n binary_search_terminal_interval_width: 10.0\n cpu_profiling_interval_sec: 1.0\n custom_report_info:\n data_resolution: Resolution(960, 540, Quarter HD)\n default_config_version: 0.30.0\n default_service_future_timeout_sec: 15.0\n enable_cpu_profiler: true\n enable_trial_buffer_preparation: false\n enforce_publisher_rate: false\n input_data_end_time: -1.0\n input_data_path: datasets/r2b_dataset/r2b_hideaway\n input_data_start_time: -1.0\n linear_scan_acceptable_frame_loss_fraction: 0.05\n linear_scan_acceptable_frame_rate_drop: 5.0\n linear_scan_duration_fraction: 1.0\n linear_scan_step_size: 5.0\n load_data_in_real_time: false\n log_file_name: kpi.json\n log_folder: /osmo/data/output\n monitor_info_list:\n - calculators:\n - class_name: BasicPerformanceCalculator\n config:\n report_prefix: ''\n module_name: ros2_benchmark.basic_performance_calculator\n service_name: start_monitoring0\n play_messages_service_future_timeout_sec: 15.0\n playback_message_buffer_size: 100\n publish_tf_messages_in_set_data: false\n publish_tf_static_messages_in_set_data: false\n publisher_lower_frequency: 10.0\n publisher_upper_frequency: 100.0\n record_data_timeline: false\n revise_timestamps_as_message_ids: false\n set_data_service_future_timeout_sec: 60.0\n setup_service_client_timeout_sec: 20.0\n start_monitoring_service_timeout_sec: 10\n start_recording_service_future_timeout_sec: 35.0\n start_recording_service_timeout_sec: 30\n test_iterations: 5\n" - } -} \ No newline at end of file diff --git a/results/stereo_image_proc_node-agx_orin.json b/results/stereo_image_proc_node-agx_orin.json index 97b0efd..6e31540 100644 --- a/results/stereo_image_proc_node-agx_orin.json +++ b/results/stereo_image_proc_node-agx_orin.json @@ -1,90 +1,111 @@ { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4980.912272135417, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 66.45005401939342, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 66.45368999738444, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0.0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 331.0, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 14.745442708333334, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 14.475911458333334, - "BasicPerformanceMetrics.MAX_JITTER": 2.80224609375, + "BasicPerformanceMetrics.RECEIVED_DURATION": 5001.140543619792, + "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 77.69998593725516, + "BasicPerformanceMetrics.MEAN_FRAME_RATE": 75.18285279902402, + "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 11.0, + "BasicPerformanceMetrics.NUM_FRAMES_SENT": 387.0, + "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 13.673746744791666, + "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 34.210367838541664, + "BasicPerformanceMetrics.MAX_JITTER": 1.783935546875, "BasicPerformanceMetrics.MIN_JITTER": 0.000244140625, - "BasicPerformanceMetrics.MEAN_JITTER": 0.19863771015387538, - "BasicPerformanceMetrics.STD_DEV_JITTER": 0.2779656200575394, - "CPUProfilingMetrics.MAX_CPU_UTIL": 26.65833333333333, - "CPUProfilingMetrics.MIN_CPU_UTIL": 25.21666666666667, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 25.875, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 0.4089415749157652, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 25.822222222222223, + "BasicPerformanceMetrics.MEAN_JITTER": 0.2838598241143048, + "BasicPerformanceMetrics.STD_DEV_JITTER": 0.270307484231264, + "ResourceMetrics.BASELINE_OVERALL_CPU_UTILIZATION": 26.805555555555557, + "ResourceMetrics.MAX_OVERALL_CPU_UTILIZATION": 45.75, + "ResourceMetrics.MIN_OVERALL_CPU_UTILIZATION": 4.166666666666667, + "ResourceMetrics.MEAN_OVERALL_CPU_UTILIZATION": 24.089488636363637, + "ResourceMetrics.STDDEV_OVERALL_CPU_UTILIZATION": 7.186135182096116, + "ResourceMetrics.BASELINE_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MAX_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MIN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_DEVICE_UTILIZATION": 0.0, "10.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4899.944091796875, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 10.203897081540946, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 10.204198060893454, + "BasicPerformanceMetrics.RECEIVED_DURATION": 4899.3818359375, + "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 10.203869119833582, + "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 14.150634765625, + "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 13.430419921875, "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, "BasicPerformanceMetrics.NUM_FRAMES_SENT": 50, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 14.860595703125, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 14.716064453125, - "BasicPerformanceMetrics.MAX_JITTER": 0.591064453125, - "BasicPerformanceMetrics.MIN_JITTER": 0.001708984375, - "BasicPerformanceMetrics.MEAN_JITTER": 0.16170756022135416, - "BasicPerformanceMetrics.STD_DEV_JITTER": 0.1248329008285954, - "CPUProfilingMetrics.MAX_CPU_UTIL": 5.683333333333334, - "CPUProfilingMetrics.MIN_CPU_UTIL": 4.808333333333334, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 5.178333333333334, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 0.3863863926750584, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 4.825 + "BasicPerformanceMetrics.MEAN_FRAME_RATE": 10.205369100494384, + "BasicPerformanceMetrics.MAX_JITTER": 0.869873046875, + "BasicPerformanceMetrics.MIN_JITTER": 0.02099609375, + "BasicPerformanceMetrics.MEAN_JITTER": 0.3548227945963542, + "BasicPerformanceMetrics.STD_DEV_JITTER": 0.21864414770246468, + "ResourceMetrics.BASELINE_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MAX_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MIN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.BASELINE_OVERALL_CPU_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_OVERALL_CPU_UTILIZATION": 3.5942622950819674, + "ResourceMetrics.STDDEV_OVERALL_CPU_UTILIZATION": 6.439253214020516, + "ResourceMetrics.MAX_OVERALL_CPU_UTILIZATION": 29.166666666666668, + "ResourceMetrics.MIN_OVERALL_CPU_UTILIZATION": 0.0 }, "60.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4983.152099609375, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 60.19970546823789, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 60.202858352149576, + "BasicPerformanceMetrics.RECEIVED_DURATION": 4983.0068359375, + "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 60.199938457712655, + "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 13.453369140625, + "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 13.06640625, "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, "BasicPerformanceMetrics.NUM_FRAMES_SENT": 300, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 14.858642578125, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 14.59765625, - "BasicPerformanceMetrics.MAX_JITTER": 8.063720703125, - "BasicPerformanceMetrics.MIN_JITTER": 0.00048828125, - "BasicPerformanceMetrics.MEAN_JITTER": 0.3096923828125, - "BasicPerformanceMetrics.STD_DEV_JITTER": 0.6977283014202155, - "CPUProfilingMetrics.MAX_CPU_UTIL": 32.40000000000001, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.8333333333333334, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 20.2125, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 9.472047998560665, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 22.433333333333334 + "BasicPerformanceMetrics.MEAN_FRAME_RATE": 60.20461337447838, + "BasicPerformanceMetrics.MAX_JITTER": 1.52294921875, + "BasicPerformanceMetrics.MIN_JITTER": 0.00146484375, + "BasicPerformanceMetrics.MEAN_JITTER": 0.26506380426803694, + "BasicPerformanceMetrics.STD_DEV_JITTER": 0.2116156518794365, + "ResourceMetrics.BASELINE_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MAX_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MIN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.BASELINE_OVERALL_CPU_UTILIZATION": 25.0, + "ResourceMetrics.MEAN_OVERALL_CPU_UTILIZATION": 19.49715909090909, + "ResourceMetrics.STDDEV_OVERALL_CPU_UTILIZATION": 6.419758317372856, + "ResourceMetrics.MAX_OVERALL_CPU_UTILIZATION": 37.5, + "ResourceMetrics.MIN_OVERALL_CPU_UTILIZATION": 4.166666666666667 }, "30.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4965.9970703125, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 30.201231929092383, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 30.205414517202044, + "BasicPerformanceMetrics.RECEIVED_DURATION": 4966.464111328125, + "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 30.200816257673523, + "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 13.37939453125, + "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 13.09033203125, "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, "BasicPerformanceMetrics.NUM_FRAMES_SENT": 150, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 15.126220703125, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 14.4384765625, - "BasicPerformanceMetrics.MAX_JITTER": 4.308349609375, - "BasicPerformanceMetrics.MIN_JITTER": 0.001220703125, - "BasicPerformanceMetrics.MEAN_JITTER": 0.25457598711993246, - "BasicPerformanceMetrics.STD_DEV_JITTER": 0.44323129560179103, - "CPUProfilingMetrics.MAX_CPU_UTIL": 12.5, - "CPUProfilingMetrics.MIN_CPU_UTIL": 11.008333333333333, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 11.788333333333332, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 0.5140417190159661, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 12.5 + "BasicPerformanceMetrics.MEAN_FRAME_RATE": 30.202574032068707, + "BasicPerformanceMetrics.MAX_JITTER": 1.027099609375, + "BasicPerformanceMetrics.MIN_JITTER": 0.0, + "BasicPerformanceMetrics.MEAN_JITTER": 0.24334056957347974, + "BasicPerformanceMetrics.STD_DEV_JITTER": 0.21542357915115268, + "ResourceMetrics.BASELINE_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MAX_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MIN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.BASELINE_OVERALL_CPU_UTILIZATION": 4.166666666666667, + "ResourceMetrics.MEAN_OVERALL_CPU_UTILIZATION": 11.012784090909092, + "ResourceMetrics.STDDEV_OVERALL_CPU_UTILIZATION": 10.057428332354846, + "ResourceMetrics.MAX_OVERALL_CPU_UTILIZATION": 38.833333333333336, + "ResourceMetrics.MIN_OVERALL_CPU_UTILIZATION": 0.0 }, "custom": { "data_resolution": "Quarter HD (960,540)" }, "metadata": { "BenchmarkMetadata.NAME": "stereo_image_proc::DisparityNode Benchmark", - "BenchmarkMetadata.TEST_FILE_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/scripts/stereo_image_proc_node.py", - "BenchmarkMetadata.TEST_DATETIME": "2023-05-25T06:31:28Z", - "BenchmarkMetadata.DEVICE_HOSTNAME": "ros-mlxi2ifxirhmtgzbkeoo64x4q4", + "BenchmarkMetadata.TEST_FILE_PATH": "/opt/ros/humble/share/ros2_benchmark/scripts/stereo_image_proc_node.py", + "BenchmarkMetadata.TEST_DATETIME": "2024-12-06T14:05:31Z", + "BenchmarkMetadata.DEVICE_HOSTNAME": "66494dc70d364ce3-270796adf8ae4ce1", "BenchmarkMetadata.DEVICE_ARCH": "aarch64", - "BenchmarkMetadata.DEVICE_OS": "Linux 5.10.104-tegra #1 SMP PREEMPT Sun Mar 19 07:55:28 PDT 2023", - "BenchmarkMetadata.IDLE_CPU_UTIL": 0.5, + "BenchmarkMetadata.DEVICE_OS": "Linux 5.15.148-tegra #1 SMP PREEMPT Thu Sep 12 21:01:54 PDT 2024", + "BenchmarkMetadata.IDLE_CPU_UTIL": 0.2777777777777778, + "BenchmarkMetadata.IDLE_GPU_UTIL": 0.0, "BenchmarkMetadata.BENCHMARK_MODE": 1, - "BenchmarkMetadata.PEAK_THROUGHPUT_PREDICTION": 66.25, - "BenchmarkMetadata.INPUT_DATA_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets/datasets/r2b_dataset/r2b_hideaway", + "BenchmarkMetadata.PEAK_THROUGHPUT_PREDICTION": 77.5, + "BenchmarkMetadata.INPUT_DATA_PATH": "/workspaces/isaac_ros-dev/src/ros2_benchmark/assets/datasets/r2b_dataset/r2b_hideaway", "BenchmarkMetadata.INPUT_DATA_SIZE": 1913946112, "BenchmarkMetadata.INPUT_DATA_HASH": "96a3c2b8007173f9da57e848b4fcee8c", - "BenchmarkMetadata.CONFIG": "ros2_benchmark_config:\n additional_fixed_publisher_rate_tests:\n - 10.0\n - 30.0\n - 60.0\n assets_root: /workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets\n benchmark_duration: 5.0\n benchmark_mode: BenchmarkMode.LOOPING\n benchmark_name: stereo_image_proc::DisparityNode Benchmark\n benchmark_namespace: r2b\n binary_search_acceptable_frame_loss_fraction: 0.05\n binary_search_acceptable_frame_rate_drop: 10.0\n binary_search_duration_fraction: 1.0\n binary_search_terminal_interval_width: 10.0\n cpu_profiling_interval_sec: 1.0\n custom_report_info:\n data_resolution: Resolution(960, 540, Quarter HD)\n default_config_version: 0.30.0\n default_service_future_timeout_sec: 15.0\n enable_cpu_profiler: true\n enable_trial_buffer_preparation: false\n enforce_publisher_rate: false\n input_data_end_time: -1.0\n input_data_path: datasets/r2b_dataset/r2b_hideaway\n input_data_start_time: -1.0\n linear_scan_acceptable_frame_loss_fraction: 0.05\n linear_scan_acceptable_frame_rate_drop: 5.0\n linear_scan_duration_fraction: 1.0\n linear_scan_step_size: 5.0\n load_data_in_real_time: false\n log_file_name: kpi.json\n log_folder: /osmo/data/output\n monitor_info_list:\n - calculators:\n - class_name: BasicPerformanceCalculator\n config:\n report_prefix: ''\n module_name: ros2_benchmark.basic_performance_calculator\n service_name: start_monitoring0\n play_messages_service_future_timeout_sec: 15.0\n playback_message_buffer_size: 50\n publish_tf_messages_in_set_data: false\n publish_tf_static_messages_in_set_data: false\n publisher_lower_frequency: 10.0\n publisher_upper_frequency: 100.0\n record_data_timeline: false\n revise_timestamps_as_message_ids: false\n set_data_service_future_timeout_sec: 60.0\n setup_service_client_timeout_sec: 20.0\n start_monitoring_service_timeout_sec: 10\n start_recording_service_future_timeout_sec: 35.0\n start_recording_service_timeout_sec: 30\n test_iterations: 5\n" + "BenchmarkMetadata.CONFIG": "ros2_benchmark_config:\n additional_fixed_publisher_rate_tests:\n - 10.0\n - 30.0\n - 60.0\n assets_root: /workspaces/isaac_ros-dev/src/ros2_benchmark/assets\n benchmark_duration: 5.0\n benchmark_mode: BenchmarkMode.LOOPING\n benchmark_name: stereo_image_proc::DisparityNode Benchmark\n benchmark_namespace: r2b\n binary_search_acceptable_frame_loss_fraction: 0.05\n binary_search_acceptable_frame_rate_drop: 10.0\n binary_search_duration_fraction: 1.0\n binary_search_terminal_interval_width: 10.0\n collect_start_timestamps_from_monitors: false\n custom_report_info:\n data_resolution: Resolution(960, 540, Quarter HD)\n default_config_version: 0.30.0\n default_service_future_timeout_sec: 15.0\n enable_resource_profiler: true\n enable_trial_buffer_preparation: false\n enforce_publisher_rate: false\n export_monitor_raw_data: false\n input_data_end_time: -1.0\n input_data_path: datasets/r2b_dataset/r2b_hideaway\n input_data_start_time: -1.0\n linear_scan_acceptable_frame_loss_fraction: 0.05\n linear_scan_acceptable_frame_rate_drop: 5.0\n linear_scan_duration_fraction: 1.0\n linear_scan_step_size: 5.0\n load_data_in_real_time: false\n log_file_name: kpi.json\n log_folder: /osmo/data/output\n monitor_info_list:\n - calculators:\n - class_name: BasicPerformanceCalculator\n config:\n report_prefix: ''\n module_name: ros2_benchmark.basic_performance_calculator\n service_name: monitor_node0\n play_messages_service_future_timeout_sec: 15.0\n playback_message_buffer_size: 50\n post_trial_run_wait_time_sec: 0.0\n pre_stop_monitoring_wait_time_sec: 5.0\n pre_trial_run_wait_time_sec: 0.0\n publish_tf_messages_in_set_data: false\n publish_tf_static_messages_in_set_data: false\n publisher_lower_frequency: 10.0\n publisher_upper_frequency: 100.0\n record_data_timeline: false\n resource_profiling_interval_sec: 0.01\n revise_timestamps_as_message_ids: false\n ros1_ws: /workspaces/isaac_ros-dev/isaac_ros_1-dev\n set_data_service_future_timeout_sec: 60.0\n setup_service_client_timeout_sec: 20.0\n start_recording_service_future_timeout_sec: 35.0\n start_recording_service_timeout_sec: 30\n test_iterations: 5\n" } } \ No newline at end of file diff --git a/results/stereo_image_proc_node-agx_xavier.json b/results/stereo_image_proc_node-agx_xavier.json deleted file mode 100644 index 6bb1a7c..0000000 --- a/results/stereo_image_proc_node-agx_xavier.json +++ /dev/null @@ -1,90 +0,0 @@ -{ - "BasicPerformanceMetrics.RECEIVED_DURATION": 4990.870524088542, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 60.82443861508549, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 57.65018249002204, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 15.333333333333334, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 303.0, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 18.410725911458332, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 27.559895833333332, - "BasicPerformanceMetrics.MAX_JITTER": 6.013916015625, - "BasicPerformanceMetrics.MIN_JITTER": 0.000244140625, - "BasicPerformanceMetrics.MEAN_JITTER": 0.8235617118969298, - "BasicPerformanceMetrics.STD_DEV_JITTER": 0.7603039808766857, - "CPUProfilingMetrics.MAX_CPU_UTIL": 41.1125, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.625, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 20.822916666666668, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 19.194657351041613, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 40.0375, - "10.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4899.710205078125, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 10.203622046679179, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 10.204685156313802, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 50, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 16.970703125, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 16.460205078125, - "BasicPerformanceMetrics.MAX_JITTER": 2.97998046875, - "BasicPerformanceMetrics.MIN_JITTER": 0.0078125, - "BasicPerformanceMetrics.MEAN_JITTER": 0.5781758626302084, - "BasicPerformanceMetrics.STD_DEV_JITTER": 0.5762641422475354, - "CPUProfilingMetrics.MAX_CPU_UTIL": 8.6875, - "CPUProfilingMetrics.MIN_CPU_UTIL": 7.9125, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 8.1875, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 0.3094349689353161, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 8.4125 - }, - "60.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 5027.661376953125, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 60.199363360378634, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 57.8797930453209, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 9, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 300, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 17.841064453125, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 62.06103515625, - "BasicPerformanceMetrics.MAX_JITTER": 8.1630859375, - "BasicPerformanceMetrics.MIN_JITTER": 0.000244140625, - "BasicPerformanceMetrics.MEAN_JITTER": 0.7568013016327855, - "BasicPerformanceMetrics.STD_DEV_JITTER": 0.8940406684688109, - "CPUProfilingMetrics.MAX_CPU_UTIL": 42.7625, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.375, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 23.921249999999997, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 17.745163117382155, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 42.7625 - }, - "30.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4965.26220703125, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 30.200192771998715, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 30.20988494577119, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 150, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 18.25439453125, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 16.660888671875, - "BasicPerformanceMetrics.MAX_JITTER": 7.425537109375, - "BasicPerformanceMetrics.MIN_JITTER": 0.015625, - "BasicPerformanceMetrics.MEAN_JITTER": 0.8319669156461149, - "BasicPerformanceMetrics.STD_DEV_JITTER": 0.9308845898811793, - "CPUProfilingMetrics.MAX_CPU_UTIL": 25.9, - "CPUProfilingMetrics.MIN_CPU_UTIL": 21.012500000000003, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 22.330000000000002, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 1.8092919885966436, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 25.9 - }, - "custom": { - "data_resolution": "Quarter HD (960,540)" - }, - "metadata": { - "BenchmarkMetadata.NAME": "stereo_image_proc::DisparityNode Benchmark", - "BenchmarkMetadata.TEST_FILE_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/scripts/stereo_image_proc_node.py", - "BenchmarkMetadata.TEST_DATETIME": "2023-05-25T06:58:18Z", - "BenchmarkMetadata.DEVICE_HOSTNAME": "ros-kevxn7zlsnhvzhfae2hwugcxdq", - "BenchmarkMetadata.DEVICE_ARCH": "aarch64", - "BenchmarkMetadata.DEVICE_OS": "Linux 5.10.104-tegra #1 SMP PREEMPT Sun Mar 19 07:55:28 PDT 2023", - "BenchmarkMetadata.IDLE_CPU_UTIL": 1.25, - "BenchmarkMetadata.BENCHMARK_MODE": 1, - "BenchmarkMetadata.PEAK_THROUGHPUT_PREDICTION": 60.625, - "BenchmarkMetadata.INPUT_DATA_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets/datasets/r2b_dataset/r2b_hideaway", - "BenchmarkMetadata.INPUT_DATA_SIZE": 1913946112, - "BenchmarkMetadata.INPUT_DATA_HASH": "96a3c2b8007173f9da57e848b4fcee8c", - "BenchmarkMetadata.CONFIG": "ros2_benchmark_config:\n additional_fixed_publisher_rate_tests:\n - 10.0\n - 30.0\n - 60.0\n assets_root: /workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets\n benchmark_duration: 5.0\n benchmark_mode: BenchmarkMode.LOOPING\n benchmark_name: stereo_image_proc::DisparityNode Benchmark\n benchmark_namespace: r2b\n binary_search_acceptable_frame_loss_fraction: 0.05\n binary_search_acceptable_frame_rate_drop: 10.0\n binary_search_duration_fraction: 1.0\n binary_search_terminal_interval_width: 10.0\n cpu_profiling_interval_sec: 1.0\n custom_report_info:\n data_resolution: Resolution(960, 540, Quarter HD)\n default_config_version: 0.30.0\n default_service_future_timeout_sec: 15.0\n enable_cpu_profiler: true\n enable_trial_buffer_preparation: false\n enforce_publisher_rate: false\n input_data_end_time: -1.0\n input_data_path: datasets/r2b_dataset/r2b_hideaway\n input_data_start_time: -1.0\n linear_scan_acceptable_frame_loss_fraction: 0.05\n linear_scan_acceptable_frame_rate_drop: 5.0\n linear_scan_duration_fraction: 1.0\n linear_scan_step_size: 5.0\n load_data_in_real_time: false\n log_file_name: kpi.json\n log_folder: /osmo/data/output\n monitor_info_list:\n - calculators:\n - class_name: BasicPerformanceCalculator\n config:\n report_prefix: ''\n module_name: ros2_benchmark.basic_performance_calculator\n service_name: start_monitoring0\n play_messages_service_future_timeout_sec: 15.0\n playback_message_buffer_size: 50\n publish_tf_messages_in_set_data: false\n publish_tf_static_messages_in_set_data: false\n publisher_lower_frequency: 10.0\n publisher_upper_frequency: 100.0\n record_data_timeline: false\n revise_timestamps_as_message_ids: false\n set_data_service_future_timeout_sec: 60.0\n setup_service_client_timeout_sec: 20.0\n start_monitoring_service_timeout_sec: 10\n start_recording_service_future_timeout_sec: 35.0\n start_recording_service_timeout_sec: 30\n test_iterations: 5\n" - } -} \ No newline at end of file diff --git a/results/stereo_image_proc_node-nuc_3060ti.json b/results/stereo_image_proc_node-nuc_3060ti.json deleted file mode 100644 index 9c3bacf..0000000 --- a/results/stereo_image_proc_node-nuc_3060ti.json +++ /dev/null @@ -1,90 +0,0 @@ -{ - "BasicPerformanceMetrics.RECEIVED_DURATION": 4982.528564453125, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 99.57393057940044, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 99.54784887118034, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0.0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 496.0, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 6.920979817708333, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 8.449055989583334, - "BasicPerformanceMetrics.MAX_JITTER": 9.97314453125, - "BasicPerformanceMetrics.MIN_JITTER": 0.002685546875, - "BasicPerformanceMetrics.MEAN_JITTER": 2.7773056956920548, - "BasicPerformanceMetrics.STD_DEV_JITTER": 2.00207953975432, - "CPUProfilingMetrics.MAX_CPU_UTIL": 17.35, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.5, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 13.707638888888889, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 5.9297186980080445, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 16.487499999999997, - "10.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4915.1357421875, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 10.202937320038094, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 10.172659031741675, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 50, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 8.698486328125, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 23.28466796875, - "BasicPerformanceMetrics.MAX_JITTER": 23.950927734375, - "BasicPerformanceMetrics.MIN_JITTER": 0.209228515625, - "BasicPerformanceMetrics.MEAN_JITTER": 9.17962646484375, - "BasicPerformanceMetrics.STD_DEV_JITTER": 6.477283267514569, - "CPUProfilingMetrics.MAX_CPU_UTIL": 5.50625, - "CPUProfilingMetrics.MIN_CPU_UTIL": 3.5, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 4.248749999999999, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 0.7311527713139024, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 3.68125 - }, - "60.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4985.49658203125, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 60.19808638866301, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 60.174547322178775, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 300, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 6.761962890625, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 8.71142578125, - "BasicPerformanceMetrics.MAX_JITTER": 23.25390625, - "BasicPerformanceMetrics.MIN_JITTER": 0.0126953125, - "BasicPerformanceMetrics.MEAN_JITTER": 6.496310854918205, - "BasicPerformanceMetrics.STD_DEV_JITTER": 4.858526621809679, - "CPUProfilingMetrics.MAX_CPU_UTIL": 13.931249999999999, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.375, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 11.330208333333331, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 4.916407236269919, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 13.55625 - }, - "30.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4972.008544921875, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 30.200079953531986, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 30.168894249629037, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 150, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 6.757080078125, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 11.891357421875, - "BasicPerformanceMetrics.MAX_JITTER": 22.269287109375, - "BasicPerformanceMetrics.MIN_JITTER": 0.028564453125, - "BasicPerformanceMetrics.MEAN_JITTER": 8.000084129539696, - "BasicPerformanceMetrics.STD_DEV_JITTER": 5.631586287390525, - "CPUProfilingMetrics.MAX_CPU_UTIL": 10.78125, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.3125, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 8.439583333333333, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 3.6533476134529304, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 10.00625 - }, - "custom": { - "data_resolution": "Quarter HD (960,540)" - }, - "metadata": { - "BenchmarkMetadata.NAME": "stereo_image_proc::DisparityNode Benchmark", - "BenchmarkMetadata.TEST_FILE_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/scripts/stereo_image_proc_node.py", - "BenchmarkMetadata.TEST_DATETIME": "2023-05-25T08:06:58Z", - "BenchmarkMetadata.DEVICE_HOSTNAME": "ros-mhi2fy7v2ndllfrbmlcccf6xla", - "BenchmarkMetadata.DEVICE_ARCH": "x86_64", - "BenchmarkMetadata.DEVICE_OS": "Linux 5.15.0-50-generic #56~20.04.1-Ubuntu SMP Tue Sep 27 15:51:29 UTC 2022", - "BenchmarkMetadata.IDLE_CPU_UTIL": 0.375, - "BenchmarkMetadata.BENCHMARK_MODE": 1, - "BenchmarkMetadata.PEAK_THROUGHPUT_PREDICTION": 99.375, - "BenchmarkMetadata.INPUT_DATA_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets/datasets/r2b_dataset/r2b_hideaway", - "BenchmarkMetadata.INPUT_DATA_SIZE": 1913946112, - "BenchmarkMetadata.INPUT_DATA_HASH": "96a3c2b8007173f9da57e848b4fcee8c", - "BenchmarkMetadata.CONFIG": "ros2_benchmark_config:\n additional_fixed_publisher_rate_tests:\n - 10.0\n - 30.0\n - 60.0\n assets_root: /workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets\n benchmark_duration: 5.0\n benchmark_mode: BenchmarkMode.LOOPING\n benchmark_name: stereo_image_proc::DisparityNode Benchmark\n benchmark_namespace: r2b\n binary_search_acceptable_frame_loss_fraction: 0.05\n binary_search_acceptable_frame_rate_drop: 10.0\n binary_search_duration_fraction: 1.0\n binary_search_terminal_interval_width: 10.0\n cpu_profiling_interval_sec: 1.0\n custom_report_info:\n data_resolution: Resolution(960, 540, Quarter HD)\n default_config_version: 0.30.0\n default_service_future_timeout_sec: 15.0\n enable_cpu_profiler: true\n enable_trial_buffer_preparation: false\n enforce_publisher_rate: false\n input_data_end_time: -1.0\n input_data_path: datasets/r2b_dataset/r2b_hideaway\n input_data_start_time: -1.0\n linear_scan_acceptable_frame_loss_fraction: 0.05\n linear_scan_acceptable_frame_rate_drop: 5.0\n linear_scan_duration_fraction: 1.0\n linear_scan_step_size: 5.0\n load_data_in_real_time: false\n log_file_name: kpi.json\n log_folder: /osmo/data/output\n monitor_info_list:\n - calculators:\n - class_name: BasicPerformanceCalculator\n config:\n report_prefix: ''\n module_name: ros2_benchmark.basic_performance_calculator\n service_name: start_monitoring0\n play_messages_service_future_timeout_sec: 15.0\n playback_message_buffer_size: 50\n publish_tf_messages_in_set_data: false\n publish_tf_static_messages_in_set_data: false\n publisher_lower_frequency: 10.0\n publisher_upper_frequency: 100.0\n record_data_timeline: false\n revise_timestamps_as_message_ids: false\n set_data_service_future_timeout_sec: 60.0\n setup_service_client_timeout_sec: 20.0\n start_monitoring_service_timeout_sec: 10\n start_recording_service_future_timeout_sec: 35.0\n start_recording_service_timeout_sec: 30\n test_iterations: 5\n" - } -} \ No newline at end of file diff --git a/results/stereo_image_proc_node-nuc_4060ti.json b/results/stereo_image_proc_node-nuc_4060ti.json deleted file mode 100644 index 48bcf28..0000000 --- a/results/stereo_image_proc_node-nuc_4060ti.json +++ /dev/null @@ -1,90 +0,0 @@ -{ - "BasicPerformanceMetrics.RECEIVED_DURATION": 4982.994791666667, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 99.5737988179372, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 99.53853490400866, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0.0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 496.0, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 6.60302734375, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 8.453694661458334, - "BasicPerformanceMetrics.MAX_JITTER": 10.169189453125, - "BasicPerformanceMetrics.MIN_JITTER": 0.010009765625, - "BasicPerformanceMetrics.MEAN_JITTER": 2.7293795072115383, - "BasicPerformanceMetrics.STD_DEV_JITTER": 1.9850807592305397, - "CPUProfilingMetrics.MAX_CPU_UTIL": 17.625, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.25, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 13.828819444444443, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 6.084520040028058, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 17.06875, - "10.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4904.591796875, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 10.203146235796304, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 10.194528325855355, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 50, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 7.13037109375, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 11.27294921875, - "BasicPerformanceMetrics.MAX_JITTER": 27.228271484375, - "BasicPerformanceMetrics.MIN_JITTER": 0.352783203125, - "BasicPerformanceMetrics.MEAN_JITTER": 9.975936889648438, - "BasicPerformanceMetrics.STD_DEV_JITTER": 6.948410271609588, - "CPUProfilingMetrics.MAX_CPU_UTIL": 4.625, - "CPUProfilingMetrics.MIN_CPU_UTIL": 3.0625, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 3.5975, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 0.551327942335594, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 3.3 - }, - "60.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4989.22607421875, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 60.19887674813977, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 60.12956629690833, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 300, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 6.48486328125, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 12.229248046875, - "BasicPerformanceMetrics.MAX_JITTER": 18.523193359375, - "BasicPerformanceMetrics.MIN_JITTER": 0.061767578125, - "BasicPerformanceMetrics.MEAN_JITTER": 5.966808344693792, - "BasicPerformanceMetrics.STD_DEV_JITTER": 4.103208601294803, - "CPUProfilingMetrics.MAX_CPU_UTIL": 13.731250000000001, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.375, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 10.971874999999999, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 4.753426327288384, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 13.731250000000001 - }, - "30.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4970.703125, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 30.19967470467579, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 30.176817288801573, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 150, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 6.42041015625, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 10.1826171875, - "BasicPerformanceMetrics.MAX_JITTER": 24.879150390625, - "BasicPerformanceMetrics.MIN_JITTER": 0.060546875, - "BasicPerformanceMetrics.MEAN_JITTER": 6.794410499366554, - "BasicPerformanceMetrics.STD_DEV_JITTER": 5.328689627995268, - "CPUProfilingMetrics.MAX_CPU_UTIL": 10.4375, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.25, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 8.238541666666666, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 3.590637087785475, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 9.425 - }, - "custom": { - "data_resolution": "Quarter HD (960,540)" - }, - "metadata": { - "BenchmarkMetadata.NAME": "stereo_image_proc::DisparityNode Benchmark", - "BenchmarkMetadata.TEST_FILE_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/scripts/stereo_image_proc_node.py", - "BenchmarkMetadata.TEST_DATETIME": "2023-05-25T06:39:51Z", - "BenchmarkMetadata.DEVICE_HOSTNAME": "ros-25bkiiqsybbpdipyva53b27rtm", - "BenchmarkMetadata.DEVICE_ARCH": "x86_64", - "BenchmarkMetadata.DEVICE_OS": "Linux 5.15.0-72-generic #79~20.04.1-Ubuntu SMP Thu Apr 20 22:12:07 UTC 2023", - "BenchmarkMetadata.IDLE_CPU_UTIL": 0.375, - "BenchmarkMetadata.BENCHMARK_MODE": 1, - "BenchmarkMetadata.PEAK_THROUGHPUT_PREDICTION": 99.375, - "BenchmarkMetadata.INPUT_DATA_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets/datasets/r2b_dataset/r2b_hideaway", - "BenchmarkMetadata.INPUT_DATA_SIZE": 1913946112, - "BenchmarkMetadata.INPUT_DATA_HASH": "96a3c2b8007173f9da57e848b4fcee8c", - "BenchmarkMetadata.CONFIG": "ros2_benchmark_config:\n additional_fixed_publisher_rate_tests:\n - 10.0\n - 30.0\n - 60.0\n assets_root: /workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets\n benchmark_duration: 5.0\n benchmark_mode: BenchmarkMode.LOOPING\n benchmark_name: stereo_image_proc::DisparityNode Benchmark\n benchmark_namespace: r2b\n binary_search_acceptable_frame_loss_fraction: 0.05\n binary_search_acceptable_frame_rate_drop: 10.0\n binary_search_duration_fraction: 1.0\n binary_search_terminal_interval_width: 10.0\n cpu_profiling_interval_sec: 1.0\n custom_report_info:\n data_resolution: Resolution(960, 540, Quarter HD)\n default_config_version: 0.30.0\n default_service_future_timeout_sec: 15.0\n enable_cpu_profiler: true\n enable_trial_buffer_preparation: false\n enforce_publisher_rate: false\n input_data_end_time: -1.0\n input_data_path: datasets/r2b_dataset/r2b_hideaway\n input_data_start_time: -1.0\n linear_scan_acceptable_frame_loss_fraction: 0.05\n linear_scan_acceptable_frame_rate_drop: 5.0\n linear_scan_duration_fraction: 1.0\n linear_scan_step_size: 5.0\n load_data_in_real_time: false\n log_file_name: kpi.json\n log_folder: /osmo/data/output\n monitor_info_list:\n - calculators:\n - class_name: BasicPerformanceCalculator\n config:\n report_prefix: ''\n module_name: ros2_benchmark.basic_performance_calculator\n service_name: start_monitoring0\n play_messages_service_future_timeout_sec: 15.0\n playback_message_buffer_size: 50\n publish_tf_messages_in_set_data: false\n publish_tf_static_messages_in_set_data: false\n publisher_lower_frequency: 10.0\n publisher_upper_frequency: 100.0\n record_data_timeline: false\n revise_timestamps_as_message_ids: false\n set_data_service_future_timeout_sec: 60.0\n setup_service_client_timeout_sec: 20.0\n start_monitoring_service_timeout_sec: 10\n start_recording_service_future_timeout_sec: 35.0\n start_recording_service_timeout_sec: 30\n test_iterations: 5\n" - } -} \ No newline at end of file diff --git a/results/stereo_image_proc_node-orin_nano.json b/results/stereo_image_proc_node-orin_nano.json index 8e9118c..fc0d55b 100644 --- a/results/stereo_image_proc_node-orin_nano.json +++ b/results/stereo_image_proc_node-orin_nano.json @@ -1,72 +1,88 @@ { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4959.607096354167, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 43.95039206912941, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 43.95509482802107, + "BasicPerformanceMetrics.RECEIVED_DURATION": 4961.6123046875, + "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 49.574946037570264, + "BasicPerformanceMetrics.MEAN_FRAME_RATE": 49.580658002995044, "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0.0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 218.0, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 22.146321614583332, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 21.650634765625, - "BasicPerformanceMetrics.MAX_JITTER": 4.29736328125, - "BasicPerformanceMetrics.MIN_JITTER": 0.002197265625, - "BasicPerformanceMetrics.MEAN_JITTER": 0.3760941116898148, - "BasicPerformanceMetrics.STD_DEV_JITTER": 0.4018516975364301, - "CPUProfilingMetrics.MAX_CPU_UTIL": 51.28333333333333, - "CPUProfilingMetrics.MIN_CPU_UTIL": 48.48333333333333, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 49.66444444444445, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 0.8087255383671875, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 49.92222222222222, + "BasicPerformanceMetrics.NUM_FRAMES_SENT": 246.0, + "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 20.552490234375, + "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 19.7822265625, + "BasicPerformanceMetrics.MAX_JITTER": 4.33837890625, + "BasicPerformanceMetrics.MIN_JITTER": 0.000244140625, + "BasicPerformanceMetrics.MEAN_JITTER": 0.48187355917008196, + "BasicPerformanceMetrics.STD_DEV_JITTER": 0.5098254996716133, + "ResourceMetrics.BASELINE_OVERALL_CPU_UTILIZATION": 41.55555555555555, + "ResourceMetrics.MAX_OVERALL_CPU_UTILIZATION": 80.5, + "ResourceMetrics.MIN_OVERALL_CPU_UTILIZATION": 16.666666666666668, + "ResourceMetrics.MEAN_OVERALL_CPU_UTILIZATION": 50.682175925925925, + "ResourceMetrics.STDDEV_OVERALL_CPU_UTILIZATION": 12.345437867318685, + "ResourceMetrics.BASELINE_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MAX_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MIN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_DEVICE_UTILIZATION": 0.0, "10.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4900.0087890625, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 10.203747615285033, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 10.204063329765232, + "BasicPerformanceMetrics.RECEIVED_DURATION": 4897.661376953125, + "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 10.2038126884909, + "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 22.31298828125, + "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 19.84521484375, "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, "BasicPerformanceMetrics.NUM_FRAMES_SENT": 50, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 21.760498046875, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 21.60888671875, - "BasicPerformanceMetrics.MAX_JITTER": 2.158935546875, - "BasicPerformanceMetrics.MIN_JITTER": 0.021728515625, - "BasicPerformanceMetrics.MEAN_JITTER": 0.4348653157552083, - "BasicPerformanceMetrics.STD_DEV_JITTER": 0.3791310831582354, - "CPUProfilingMetrics.MAX_CPU_UTIL": 12.216666666666667, - "CPUProfilingMetrics.MIN_CPU_UTIL": 9.4, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 10.713333333333335, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 1.170251634858456, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 12.216666666666667 + "BasicPerformanceMetrics.MEAN_FRAME_RATE": 10.208954060255062, + "BasicPerformanceMetrics.MAX_JITTER": 7.98974609375, + "BasicPerformanceMetrics.MIN_JITTER": 0.003662109375, + "BasicPerformanceMetrics.MEAN_JITTER": 0.9963124593098959, + "BasicPerformanceMetrics.STD_DEV_JITTER": 1.2228817160474692, + "ResourceMetrics.BASELINE_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MAX_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MIN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.BASELINE_OVERALL_CPU_UTILIZATION": 16.666666666666668, + "ResourceMetrics.MEAN_OVERALL_CPU_UTILIZATION": 10.504166666666666, + "ResourceMetrics.STDDEV_OVERALL_CPU_UTILIZATION": 18.02007577084617, + "ResourceMetrics.MAX_OVERALL_CPU_UTILIZATION": 66.66666666666667, + "ResourceMetrics.MIN_OVERALL_CPU_UTILIZATION": 0.0 }, "30.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4966.644287109375, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 30.200362001279288, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 30.20147836826485, + "BasicPerformanceMetrics.RECEIVED_DURATION": 4965.982421875, + "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 30.200503027128548, + "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 20.6376953125, + "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 19.8154296875, "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, "BasicPerformanceMetrics.NUM_FRAMES_SENT": 150, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 21.981201171875, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 21.797607421875, - "BasicPerformanceMetrics.MAX_JITTER": 2.02783203125, - "BasicPerformanceMetrics.MIN_JITTER": 0.0009765625, - "BasicPerformanceMetrics.MEAN_JITTER": 0.35172745988175674, - "BasicPerformanceMetrics.STD_DEV_JITTER": 0.2963570818534654, - "CPUProfilingMetrics.MAX_CPU_UTIL": 37.06666666666666, - "CPUProfilingMetrics.MIN_CPU_UTIL": 34.71666666666666, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 35.81333333333333, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 0.8982328329682815, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 37.06666666666666 + "BasicPerformanceMetrics.MEAN_FRAME_RATE": 30.205503615811164, + "BasicPerformanceMetrics.MAX_JITTER": 1.935791015625, + "BasicPerformanceMetrics.MIN_JITTER": 0.00146484375, + "BasicPerformanceMetrics.MEAN_JITTER": 0.5220468881967906, + "BasicPerformanceMetrics.STD_DEV_JITTER": 0.38384982280217766, + "ResourceMetrics.BASELINE_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MAX_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MIN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.BASELINE_OVERALL_CPU_UTILIZATION": 8.333333333333334, + "ResourceMetrics.MEAN_OVERALL_CPU_UTILIZATION": 30.624305555555555, + "ResourceMetrics.STDDEV_OVERALL_CPU_UTILIZATION": 20.660711215823934, + "ResourceMetrics.MAX_OVERALL_CPU_UTILIZATION": 80.5, + "ResourceMetrics.MIN_OVERALL_CPU_UTILIZATION": 0.0 }, "custom": { "data_resolution": "Quarter HD (960,540)" }, "metadata": { "BenchmarkMetadata.NAME": "stereo_image_proc::DisparityNode Benchmark", - "BenchmarkMetadata.TEST_FILE_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/scripts/stereo_image_proc_node.py", - "BenchmarkMetadata.TEST_DATETIME": "2023-05-25T06:26:17Z", - "BenchmarkMetadata.DEVICE_HOSTNAME": "ros-xny5diobvvbjvl6p7sz4djgmyq", + "BenchmarkMetadata.TEST_FILE_PATH": "/opt/ros/humble/share/ros2_benchmark/scripts/stereo_image_proc_node.py", + "BenchmarkMetadata.TEST_DATETIME": "2024-12-06T07:24:17Z", + "BenchmarkMetadata.DEVICE_HOSTNAME": "64828c8415d34b1d-06edff4e69184de9", "BenchmarkMetadata.DEVICE_ARCH": "aarch64", - "BenchmarkMetadata.DEVICE_OS": "Linux 5.10.104-tegra #1 SMP PREEMPT Wed Feb 1 14:17:37 PST 2023", - "BenchmarkMetadata.IDLE_CPU_UTIL": 21.46666666666667, + "BenchmarkMetadata.DEVICE_OS": "Linux 5.15.148-tegra #1 SMP PREEMPT Thu Sep 12 21:01:54 PDT 2024", + "BenchmarkMetadata.IDLE_CPU_UTIL": 0.8611111111111112, + "BenchmarkMetadata.IDLE_GPU_UTIL": 0.0, "BenchmarkMetadata.BENCHMARK_MODE": 1, - "BenchmarkMetadata.PEAK_THROUGHPUT_PREDICTION": 43.75, - "BenchmarkMetadata.INPUT_DATA_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets/datasets/r2b_dataset/r2b_hideaway", + "BenchmarkMetadata.PEAK_THROUGHPUT_PREDICTION": 49.375, + "BenchmarkMetadata.INPUT_DATA_PATH": "/workspaces/isaac_ros-dev/src/ros2_benchmark/assets/datasets/r2b_dataset/r2b_hideaway", "BenchmarkMetadata.INPUT_DATA_SIZE": 1913946112, "BenchmarkMetadata.INPUT_DATA_HASH": "96a3c2b8007173f9da57e848b4fcee8c", - "BenchmarkMetadata.CONFIG": "ros2_benchmark_config:\n additional_fixed_publisher_rate_tests:\n - 10.0\n - 30.0\n - 60.0\n assets_root: /workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets\n benchmark_duration: 5.0\n benchmark_mode: BenchmarkMode.LOOPING\n benchmark_name: stereo_image_proc::DisparityNode Benchmark\n benchmark_namespace: r2b\n binary_search_acceptable_frame_loss_fraction: 0.05\n binary_search_acceptable_frame_rate_drop: 10.0\n binary_search_duration_fraction: 1.0\n binary_search_terminal_interval_width: 10.0\n cpu_profiling_interval_sec: 1.0\n custom_report_info:\n data_resolution: Resolution(960, 540, Quarter HD)\n default_config_version: 0.30.0\n default_service_future_timeout_sec: 15.0\n enable_cpu_profiler: true\n enable_trial_buffer_preparation: false\n enforce_publisher_rate: false\n input_data_end_time: -1.0\n input_data_path: datasets/r2b_dataset/r2b_hideaway\n input_data_start_time: -1.0\n linear_scan_acceptable_frame_loss_fraction: 0.05\n linear_scan_acceptable_frame_rate_drop: 5.0\n linear_scan_duration_fraction: 1.0\n linear_scan_step_size: 5.0\n load_data_in_real_time: false\n log_file_name: kpi.json\n log_folder: /osmo/data/output\n monitor_info_list:\n - calculators:\n - class_name: BasicPerformanceCalculator\n config:\n report_prefix: ''\n module_name: ros2_benchmark.basic_performance_calculator\n service_name: start_monitoring0\n play_messages_service_future_timeout_sec: 15.0\n playback_message_buffer_size: 50\n publish_tf_messages_in_set_data: false\n publish_tf_static_messages_in_set_data: false\n publisher_lower_frequency: 10.0\n publisher_upper_frequency: 100.0\n record_data_timeline: false\n revise_timestamps_as_message_ids: false\n set_data_service_future_timeout_sec: 60.0\n setup_service_client_timeout_sec: 20.0\n start_monitoring_service_timeout_sec: 10\n start_recording_service_future_timeout_sec: 35.0\n start_recording_service_timeout_sec: 30\n test_iterations: 5\n" + "BenchmarkMetadata.CONFIG": "ros2_benchmark_config:\n additional_fixed_publisher_rate_tests:\n - 10.0\n - 30.0\n - 60.0\n assets_root: /workspaces/isaac_ros-dev/src/ros2_benchmark/assets\n benchmark_duration: 5.0\n benchmark_mode: BenchmarkMode.LOOPING\n benchmark_name: stereo_image_proc::DisparityNode Benchmark\n benchmark_namespace: r2b\n binary_search_acceptable_frame_loss_fraction: 0.05\n binary_search_acceptable_frame_rate_drop: 10.0\n binary_search_duration_fraction: 1.0\n binary_search_terminal_interval_width: 10.0\n collect_start_timestamps_from_monitors: false\n custom_report_info:\n data_resolution: Resolution(960, 540, Quarter HD)\n default_config_version: 0.30.0\n default_service_future_timeout_sec: 15.0\n enable_resource_profiler: true\n enable_trial_buffer_preparation: false\n enforce_publisher_rate: false\n export_monitor_raw_data: false\n input_data_end_time: -1.0\n input_data_path: datasets/r2b_dataset/r2b_hideaway\n input_data_start_time: -1.0\n linear_scan_acceptable_frame_loss_fraction: 0.05\n linear_scan_acceptable_frame_rate_drop: 5.0\n linear_scan_duration_fraction: 1.0\n linear_scan_step_size: 5.0\n load_data_in_real_time: false\n log_file_name: kpi.json\n log_folder: /osmo/data/output\n monitor_info_list:\n - calculators:\n - class_name: BasicPerformanceCalculator\n config:\n report_prefix: ''\n module_name: ros2_benchmark.basic_performance_calculator\n service_name: monitor_node0\n play_messages_service_future_timeout_sec: 15.0\n playback_message_buffer_size: 50\n post_trial_run_wait_time_sec: 0.0\n pre_stop_monitoring_wait_time_sec: 5.0\n pre_trial_run_wait_time_sec: 0.0\n publish_tf_messages_in_set_data: false\n publish_tf_static_messages_in_set_data: false\n publisher_lower_frequency: 10.0\n publisher_upper_frequency: 100.0\n record_data_timeline: false\n resource_profiling_interval_sec: 0.01\n revise_timestamps_as_message_ids: false\n ros1_ws: /workspaces/isaac_ros-dev/isaac_ros_1-dev\n set_data_service_future_timeout_sec: 60.0\n setup_service_client_timeout_sec: 20.0\n start_recording_service_future_timeout_sec: 35.0\n start_recording_service_timeout_sec: 30\n test_iterations: 5\n" } } \ No newline at end of file diff --git a/results/stereo_image_proc_node-orin_nano_emul.json b/results/stereo_image_proc_node-orin_nano_emul.json deleted file mode 100644 index 9abd727..0000000 --- a/results/stereo_image_proc_node-orin_nano_emul.json +++ /dev/null @@ -1,72 +0,0 @@ -{ - "BasicPerformanceMetrics.RECEIVED_DURATION": 4993.79248046875, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 49.57558264599402, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 48.593703137009726, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 3.3333333333333335, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 246.0, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 20.727864583333332, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 52.76171875, - "BasicPerformanceMetrics.MAX_JITTER": 10.210693359375, - "BasicPerformanceMetrics.MIN_JITTER": 0.000244140625, - "BasicPerformanceMetrics.MEAN_JITTER": 1.2354741393283195, - "BasicPerformanceMetrics.STD_DEV_JITTER": 1.130872757825277, - "CPUProfilingMetrics.MAX_CPU_UTIL": 54.15, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.5, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 26.262777777777774, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 24.851687833438977, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 50.211111111111116, - "10.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4899.375244140625, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 10.203699319301654, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 10.205382831167539, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 50, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 21.604248046875, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 20.7958984375, - "BasicPerformanceMetrics.MAX_JITTER": 5.24267578125, - "BasicPerformanceMetrics.MIN_JITTER": 0.11865234375, - "BasicPerformanceMetrics.MEAN_JITTER": 1.421051025390625, - "BasicPerformanceMetrics.STD_DEV_JITTER": 1.1715330595922302, - "CPUProfilingMetrics.MAX_CPU_UTIL": 13.166666666666666, - "CPUProfilingMetrics.MIN_CPU_UTIL": 11.25, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 12.376666666666665, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 0.7333787864701596, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 12.916666666666666 - }, - "30.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4965.4111328125, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 30.201948989183435, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 30.208978871612036, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 150, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 21.62255859375, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 20.466796875, - "BasicPerformanceMetrics.MAX_JITTER": 5.56689453125, - "BasicPerformanceMetrics.MIN_JITTER": 0.016357421875, - "BasicPerformanceMetrics.MEAN_JITTER": 2.006065574852196, - "BasicPerformanceMetrics.STD_DEV_JITTER": 1.6358925727170521, - "CPUProfilingMetrics.MAX_CPU_UTIL": 31.13333333333333, - "CPUProfilingMetrics.MIN_CPU_UTIL": 29.7, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 30.316666666666663, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 0.47340375063059, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 30.349999999999998 - }, - "custom": { - "data_resolution": "Quarter HD (960,540)" - }, - "metadata": { - "BenchmarkMetadata.NAME": "stereo_image_proc::DisparityNode Benchmark", - "BenchmarkMetadata.TEST_FILE_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/scripts/stereo_image_proc_node.py", - "BenchmarkMetadata.TEST_DATETIME": "2023-05-25T06:38:02Z", - "BenchmarkMetadata.DEVICE_HOSTNAME": "ros-qn2fnyqzzbg3hkflcjco4kwciu", - "BenchmarkMetadata.DEVICE_ARCH": "aarch64", - "BenchmarkMetadata.DEVICE_OS": "Linux 5.10.104-tegra #1 SMP PREEMPT Sun Mar 19 07:55:28 PDT 2023", - "BenchmarkMetadata.IDLE_CPU_UTIL": 1.1666666666666667, - "BenchmarkMetadata.BENCHMARK_MODE": 1, - "BenchmarkMetadata.PEAK_THROUGHPUT_PREDICTION": 49.375, - "BenchmarkMetadata.INPUT_DATA_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets/datasets/r2b_dataset/r2b_hideaway", - "BenchmarkMetadata.INPUT_DATA_SIZE": 1913946112, - "BenchmarkMetadata.INPUT_DATA_HASH": "96a3c2b8007173f9da57e848b4fcee8c", - "BenchmarkMetadata.CONFIG": "ros2_benchmark_config:\n additional_fixed_publisher_rate_tests:\n - 10.0\n - 30.0\n - 60.0\n assets_root: /workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets\n benchmark_duration: 5.0\n benchmark_mode: BenchmarkMode.LOOPING\n benchmark_name: stereo_image_proc::DisparityNode Benchmark\n benchmark_namespace: r2b\n binary_search_acceptable_frame_loss_fraction: 0.05\n binary_search_acceptable_frame_rate_drop: 10.0\n binary_search_duration_fraction: 1.0\n binary_search_terminal_interval_width: 10.0\n cpu_profiling_interval_sec: 1.0\n custom_report_info:\n data_resolution: Resolution(960, 540, Quarter HD)\n default_config_version: 0.30.0\n default_service_future_timeout_sec: 15.0\n enable_cpu_profiler: true\n enable_trial_buffer_preparation: false\n enforce_publisher_rate: false\n input_data_end_time: -1.0\n input_data_path: datasets/r2b_dataset/r2b_hideaway\n input_data_start_time: -1.0\n linear_scan_acceptable_frame_loss_fraction: 0.05\n linear_scan_acceptable_frame_rate_drop: 5.0\n linear_scan_duration_fraction: 1.0\n linear_scan_step_size: 5.0\n load_data_in_real_time: false\n log_file_name: kpi.json\n log_folder: /osmo/data/output\n monitor_info_list:\n - calculators:\n - class_name: BasicPerformanceCalculator\n config:\n report_prefix: ''\n module_name: ros2_benchmark.basic_performance_calculator\n service_name: start_monitoring0\n play_messages_service_future_timeout_sec: 15.0\n playback_message_buffer_size: 50\n publish_tf_messages_in_set_data: false\n publish_tf_static_messages_in_set_data: false\n publisher_lower_frequency: 10.0\n publisher_upper_frequency: 100.0\n record_data_timeline: false\n revise_timestamps_as_message_ids: false\n set_data_service_future_timeout_sec: 60.0\n setup_service_client_timeout_sec: 20.0\n start_monitoring_service_timeout_sec: 10\n start_recording_service_future_timeout_sec: 35.0\n start_recording_service_timeout_sec: 30\n test_iterations: 5\n" - } -} \ No newline at end of file diff --git a/results/stereo_image_proc_node-orin_nx.json b/results/stereo_image_proc_node-orin_nx.json index 60537e7..3d87849 100644 --- a/results/stereo_image_proc_node-orin_nx.json +++ b/results/stereo_image_proc_node-orin_nx.json @@ -1,90 +1,111 @@ { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4981.489664713542, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 60.824249826805776, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 60.82517930090044, + "BasicPerformanceMetrics.RECEIVED_DURATION": 4980.828857421875, + "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 66.44974895921541, + "BasicPerformanceMetrics.MEAN_FRAME_RATE": 66.45480291097364, "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0.0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 303.0, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 16.649332682291668, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 16.51904296875, - "BasicPerformanceMetrics.MAX_JITTER": 2.1474609375, - "BasicPerformanceMetrics.MIN_JITTER": 0.000244140625, - "BasicPerformanceMetrics.MEAN_JITTER": 0.7127194832329734, - "BasicPerformanceMetrics.STD_DEV_JITTER": 0.5266721507469957, - "CPUProfilingMetrics.MAX_CPU_UTIL": 40.262499999999996, - "CPUProfilingMetrics.MIN_CPU_UTIL": 38.6875, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 39.54416666666666, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 0.5548026598597581, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 39.791666666666664, + "BasicPerformanceMetrics.NUM_FRAMES_SENT": 331.0, + "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 14.847981770833334, + "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 14.469156901041666, + "BasicPerformanceMetrics.MAX_JITTER": 4.478271484375, + "BasicPerformanceMetrics.MIN_JITTER": 0.000732421875, + "BasicPerformanceMetrics.MEAN_JITTER": 0.33219302099164133, + "BasicPerformanceMetrics.STD_DEV_JITTER": 0.3436907167265199, + "ResourceMetrics.BASELINE_OVERALL_CPU_UTILIZATION": 36.708333333333336, + "ResourceMetrics.MAX_OVERALL_CPU_UTILIZATION": 70.5, + "ResourceMetrics.MIN_OVERALL_CPU_UTILIZATION": 12.5, + "ResourceMetrics.MEAN_OVERALL_CPU_UTILIZATION": 36.705078125, + "ResourceMetrics.STDDEV_OVERALL_CPU_UTILIZATION": 9.02248537588743, + "ResourceMetrics.BASELINE_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MAX_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MIN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_DEVICE_UTILIZATION": 0.0, "10.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4899.81005859375, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 10.203755749390377, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 10.204477194438441, + "BasicPerformanceMetrics.RECEIVED_DURATION": 4899.601318359375, + "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 10.203776593094515, + "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 15.09521484375, + "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 14.550048828125, "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, "BasicPerformanceMetrics.NUM_FRAMES_SENT": 50, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 16.517822265625, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 16.17138671875, - "BasicPerformanceMetrics.MAX_JITTER": 1.767333984375, - "BasicPerformanceMetrics.MIN_JITTER": 0.01416015625, - "BasicPerformanceMetrics.MEAN_JITTER": 0.5571034749348959, - "BasicPerformanceMetrics.STD_DEV_JITTER": 0.49218649899311134, - "CPUProfilingMetrics.MAX_CPU_UTIL": 9.125, - "CPUProfilingMetrics.MIN_CPU_UTIL": 8.662500000000001, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 8.7775, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 0.17453509675707046, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 8.6875 + "BasicPerformanceMetrics.MEAN_FRAME_RATE": 10.204911941027568, + "BasicPerformanceMetrics.MAX_JITTER": 0.998046875, + "BasicPerformanceMetrics.MIN_JITTER": 0.006103515625, + "BasicPerformanceMetrics.MEAN_JITTER": 0.3606669108072917, + "BasicPerformanceMetrics.STD_DEV_JITTER": 0.24106192243744667, + "ResourceMetrics.BASELINE_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MAX_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MIN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.BASELINE_OVERALL_CPU_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_OVERALL_CPU_UTILIZATION": 5.361778846153846, + "ResourceMetrics.STDDEV_OVERALL_CPU_UTILIZATION": 9.820747976915555, + "ResourceMetrics.MAX_OVERALL_CPU_UTILIZATION": 47.875, + "ResourceMetrics.MIN_OVERALL_CPU_UTILIZATION": 0.0 }, "60.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4983.537353515625, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 60.20013310852999, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 60.19820435144641, + "BasicPerformanceMetrics.RECEIVED_DURATION": 4983.043701171875, + "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 60.20106214113544, + "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 14.939697265625, + "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 14.6826171875, "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, "BasicPerformanceMetrics.NUM_FRAMES_SENT": 300, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 16.35498046875, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 16.5146484375, - "BasicPerformanceMetrics.MAX_JITTER": 2.880126953125, - "BasicPerformanceMetrics.MIN_JITTER": 0.0029296875, - "BasicPerformanceMetrics.MEAN_JITTER": 0.7143898778313759, - "BasicPerformanceMetrics.STD_DEV_JITTER": 0.5670632526326947, - "CPUProfilingMetrics.MAX_CPU_UTIL": 40.0625, - "CPUProfilingMetrics.MIN_CPU_UTIL": 38.5, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 39.182500000000005, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 0.5573710613227069, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 39.5625 + "BasicPerformanceMetrics.MEAN_FRAME_RATE": 60.20416797256831, + "BasicPerformanceMetrics.MAX_JITTER": 4.077392578125, + "BasicPerformanceMetrics.MIN_JITTER": 0.00146484375, + "BasicPerformanceMetrics.MEAN_JITTER": 0.3557145291526846, + "BasicPerformanceMetrics.STD_DEV_JITTER": 0.4473862538275355, + "ResourceMetrics.BASELINE_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MAX_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MIN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.BASELINE_OVERALL_CPU_UTILIZATION": 43.625, + "ResourceMetrics.MEAN_OVERALL_CPU_UTILIZATION": 33.298828125, + "ResourceMetrics.STDDEV_OVERALL_CPU_UTILIZATION": 9.313171567319802, + "ResourceMetrics.MAX_OVERALL_CPU_UTILIZATION": 60.375, + "ResourceMetrics.MIN_OVERALL_CPU_UTILIZATION": 12.5 }, "30.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4966.4560546875, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 30.200739062812673, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 30.2026230270225, + "BasicPerformanceMetrics.RECEIVED_DURATION": 4966.6259765625, + "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 30.200886030291134, + "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 14.826416015625, + "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 14.710693359375, "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, "BasicPerformanceMetrics.NUM_FRAMES_SENT": 150, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 16.501708984375, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 16.19189453125, - "BasicPerformanceMetrics.MAX_JITTER": 1.8427734375, - "BasicPerformanceMetrics.MIN_JITTER": 0.001220703125, - "BasicPerformanceMetrics.MEAN_JITTER": 0.5947034681165541, - "BasicPerformanceMetrics.STD_DEV_JITTER": 0.472665956066369, - "CPUProfilingMetrics.MAX_CPU_UTIL": 19.1125, - "CPUProfilingMetrics.MIN_CPU_UTIL": 18.1125, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 18.4775, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 0.37059074462269065, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 18.1125 + "BasicPerformanceMetrics.MEAN_FRAME_RATE": 30.201589712583505, + "BasicPerformanceMetrics.MAX_JITTER": 1.737548828125, + "BasicPerformanceMetrics.MIN_JITTER": 0.001708984375, + "BasicPerformanceMetrics.MEAN_JITTER": 0.27293272276182434, + "BasicPerformanceMetrics.STD_DEV_JITTER": 0.25598957936866384, + "ResourceMetrics.BASELINE_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MAX_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MIN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.BASELINE_OVERALL_CPU_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_OVERALL_CPU_UTILIZATION": 17.736328125, + "ResourceMetrics.STDDEV_OVERALL_CPU_UTILIZATION": 15.287438843283255, + "ResourceMetrics.MAX_OVERALL_CPU_UTILIZATION": 62.5, + "ResourceMetrics.MIN_OVERALL_CPU_UTILIZATION": 0.0 }, "custom": { "data_resolution": "Quarter HD (960,540)" }, "metadata": { "BenchmarkMetadata.NAME": "stereo_image_proc::DisparityNode Benchmark", - "BenchmarkMetadata.TEST_FILE_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/scripts/stereo_image_proc_node.py", - "BenchmarkMetadata.TEST_DATETIME": "2023-05-25T06:17:28Z", - "BenchmarkMetadata.DEVICE_HOSTNAME": "ros-htumaf67evfmvi7jishf2ck4h4", + "BenchmarkMetadata.TEST_FILE_PATH": "/opt/ros/humble/share/ros2_benchmark/scripts/stereo_image_proc_node.py", + "BenchmarkMetadata.TEST_DATETIME": "2024-12-06T08:24:38Z", + "BenchmarkMetadata.DEVICE_HOSTNAME": "a7c67ef73cf942c5-598131dfe8fe496d", "BenchmarkMetadata.DEVICE_ARCH": "aarch64", - "BenchmarkMetadata.DEVICE_OS": "Linux 5.10.104-tegra #1 SMP PREEMPT Wed Mar 15 00:17:07 PDT 2023", - "BenchmarkMetadata.IDLE_CPU_UTIL": 0.375, + "BenchmarkMetadata.DEVICE_OS": "Linux 5.15.148-tegra #1 SMP PREEMPT Thu Sep 12 21:01:54 PDT 2024", + "BenchmarkMetadata.IDLE_CPU_UTIL": 0.5625, + "BenchmarkMetadata.IDLE_GPU_UTIL": 0.0, "BenchmarkMetadata.BENCHMARK_MODE": 1, - "BenchmarkMetadata.PEAK_THROUGHPUT_PREDICTION": 60.625, - "BenchmarkMetadata.INPUT_DATA_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets/datasets/r2b_dataset/r2b_hideaway", + "BenchmarkMetadata.PEAK_THROUGHPUT_PREDICTION": 66.25, + "BenchmarkMetadata.INPUT_DATA_PATH": "/workspaces/isaac_ros-dev/src/ros2_benchmark/assets/datasets/r2b_dataset/r2b_hideaway", "BenchmarkMetadata.INPUT_DATA_SIZE": 1913946112, "BenchmarkMetadata.INPUT_DATA_HASH": "96a3c2b8007173f9da57e848b4fcee8c", - "BenchmarkMetadata.CONFIG": "ros2_benchmark_config:\n additional_fixed_publisher_rate_tests:\n - 10.0\n - 30.0\n - 60.0\n assets_root: /workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets\n benchmark_duration: 5.0\n benchmark_mode: BenchmarkMode.LOOPING\n benchmark_name: stereo_image_proc::DisparityNode Benchmark\n benchmark_namespace: r2b\n binary_search_acceptable_frame_loss_fraction: 0.05\n binary_search_acceptable_frame_rate_drop: 10.0\n binary_search_duration_fraction: 1.0\n binary_search_terminal_interval_width: 10.0\n cpu_profiling_interval_sec: 1.0\n custom_report_info:\n data_resolution: Resolution(960, 540, Quarter HD)\n default_config_version: 0.30.0\n default_service_future_timeout_sec: 15.0\n enable_cpu_profiler: true\n enable_trial_buffer_preparation: false\n enforce_publisher_rate: false\n input_data_end_time: -1.0\n input_data_path: datasets/r2b_dataset/r2b_hideaway\n input_data_start_time: -1.0\n linear_scan_acceptable_frame_loss_fraction: 0.05\n linear_scan_acceptable_frame_rate_drop: 5.0\n linear_scan_duration_fraction: 1.0\n linear_scan_step_size: 5.0\n load_data_in_real_time: false\n log_file_name: kpi.json\n log_folder: /osmo/data/output\n monitor_info_list:\n - calculators:\n - class_name: BasicPerformanceCalculator\n config:\n report_prefix: ''\n module_name: ros2_benchmark.basic_performance_calculator\n service_name: start_monitoring0\n play_messages_service_future_timeout_sec: 15.0\n playback_message_buffer_size: 50\n publish_tf_messages_in_set_data: false\n publish_tf_static_messages_in_set_data: false\n publisher_lower_frequency: 10.0\n publisher_upper_frequency: 100.0\n record_data_timeline: false\n revise_timestamps_as_message_ids: false\n set_data_service_future_timeout_sec: 60.0\n setup_service_client_timeout_sec: 20.0\n start_monitoring_service_timeout_sec: 10\n start_recording_service_future_timeout_sec: 35.0\n start_recording_service_timeout_sec: 30\n test_iterations: 5\n" + "BenchmarkMetadata.CONFIG": "ros2_benchmark_config:\n additional_fixed_publisher_rate_tests:\n - 10.0\n - 30.0\n - 60.0\n assets_root: /workspaces/isaac_ros-dev/src/ros2_benchmark/assets\n benchmark_duration: 5.0\n benchmark_mode: BenchmarkMode.LOOPING\n benchmark_name: stereo_image_proc::DisparityNode Benchmark\n benchmark_namespace: r2b\n binary_search_acceptable_frame_loss_fraction: 0.05\n binary_search_acceptable_frame_rate_drop: 10.0\n binary_search_duration_fraction: 1.0\n binary_search_terminal_interval_width: 10.0\n collect_start_timestamps_from_monitors: false\n custom_report_info:\n data_resolution: Resolution(960, 540, Quarter HD)\n default_config_version: 0.30.0\n default_service_future_timeout_sec: 15.0\n enable_resource_profiler: true\n enable_trial_buffer_preparation: false\n enforce_publisher_rate: false\n export_monitor_raw_data: false\n input_data_end_time: -1.0\n input_data_path: datasets/r2b_dataset/r2b_hideaway\n input_data_start_time: -1.0\n linear_scan_acceptable_frame_loss_fraction: 0.05\n linear_scan_acceptable_frame_rate_drop: 5.0\n linear_scan_duration_fraction: 1.0\n linear_scan_step_size: 5.0\n load_data_in_real_time: false\n log_file_name: kpi.json\n log_folder: /osmo/data/output\n monitor_info_list:\n - calculators:\n - class_name: BasicPerformanceCalculator\n config:\n report_prefix: ''\n module_name: ros2_benchmark.basic_performance_calculator\n service_name: monitor_node0\n play_messages_service_future_timeout_sec: 15.0\n playback_message_buffer_size: 50\n post_trial_run_wait_time_sec: 0.0\n pre_stop_monitoring_wait_time_sec: 5.0\n pre_trial_run_wait_time_sec: 0.0\n publish_tf_messages_in_set_data: false\n publish_tf_static_messages_in_set_data: false\n publisher_lower_frequency: 10.0\n publisher_upper_frequency: 100.0\n record_data_timeline: false\n resource_profiling_interval_sec: 0.01\n revise_timestamps_as_message_ids: false\n ros1_ws: /workspaces/isaac_ros-dev/isaac_ros_1-dev\n set_data_service_future_timeout_sec: 60.0\n setup_service_client_timeout_sec: 20.0\n start_recording_service_future_timeout_sec: 35.0\n start_recording_service_timeout_sec: 30\n test_iterations: 5\n" } } \ No newline at end of file diff --git a/results/stereo_image_proc_node-x86-4090.json b/results/stereo_image_proc_node-x86-4090.json new file mode 100644 index 0000000..d429cde --- /dev/null +++ b/results/stereo_image_proc_node-x86-4090.json @@ -0,0 +1,131 @@ +{ + "BasicPerformanceMetrics.RECEIVED_DURATION": 4981.308024088542, + "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 99.5759022728699, + "BasicPerformanceMetrics.MEAN_FRAME_RATE": 99.57224783784208, + "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0.0, + "BasicPerformanceMetrics.NUM_FRAMES_SENT": 496.0, + "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 6.39892578125, + "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 6.571858723958333, + "BasicPerformanceMetrics.MAX_JITTER": 5.4970703125, + "BasicPerformanceMetrics.MIN_JITTER": 0.003662109375, + "BasicPerformanceMetrics.MEAN_JITTER": 1.9253636410361843, + "BasicPerformanceMetrics.STD_DEV_JITTER": 1.220541405047853, + "ResourceMetrics.BASELINE_OVERALL_CPU_UTILIZATION": 3.8197916666666667, + "ResourceMetrics.MAX_OVERALL_CPU_UTILIZATION": 17.1875, + "ResourceMetrics.MIN_OVERALL_CPU_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_OVERALL_CPU_UTILIZATION": 2.540964094725863, + "ResourceMetrics.STDDEV_OVERALL_CPU_UTILIZATION": 3.4383036875644897, + "ResourceMetrics.BASELINE_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MAX_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MIN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.BASELINE_DEVICE_MEMORY_UTILIZATION": 1.9400000000000002, + "ResourceMetrics.MAX_DEVICE_MEMORY_UTILIZATION": 1.94, + "ResourceMetrics.MIN_DEVICE_MEMORY_UTILIZATION": 1.94, + "ResourceMetrics.MEAN_DEVICE_MEMORY_UTILIZATION": 1.9400000000000004, + "ResourceMetrics.STDDEV_DEVICE_MEMORY_UTILIZATION": 3.7007434154171886e-16, + "10.0fps": { + "BasicPerformanceMetrics.RECEIVED_DURATION": 4898.525390625, + "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 10.204180774436534, + "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 7.367431640625, + "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 5.9404296875, + "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, + "BasicPerformanceMetrics.NUM_FRAMES_SENT": 50, + "BasicPerformanceMetrics.MEAN_FRAME_RATE": 10.207153380421802, + "BasicPerformanceMetrics.MAX_JITTER": 4.360107421875, + "BasicPerformanceMetrics.MIN_JITTER": 0.059326171875, + "BasicPerformanceMetrics.MEAN_JITTER": 1.6701253255208333, + "BasicPerformanceMetrics.STD_DEV_JITTER": 1.131224471024291, + "ResourceMetrics.BASELINE_OVERALL_CPU_UTILIZATION": 14.0625, + "ResourceMetrics.MEAN_OVERALL_CPU_UTILIZATION": 0.4312298387096774, + "ResourceMetrics.MAX_OVERALL_CPU_UTILIZATION": 14.0625, + "ResourceMetrics.MIN_OVERALL_CPU_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_OVERALL_CPU_UTILIZATION": 1.7178073965895193, + "ResourceMetrics.BASELINE_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MAX_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MIN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.BASELINE_DEVICE_MEMORY_UTILIZATION": 1.94, + "ResourceMetrics.MEAN_DEVICE_MEMORY_UTILIZATION": 1.9400000000000004, + "ResourceMetrics.STDDEV_DEVICE_MEMORY_UTILIZATION": 4.440892098500626e-16, + "ResourceMetrics.MAX_DEVICE_MEMORY_UTILIZATION": 1.94, + "ResourceMetrics.MIN_DEVICE_MEMORY_UTILIZATION": 1.94 + }, + "60.0fps": { + "BasicPerformanceMetrics.RECEIVED_DURATION": 4985.003173828125, + "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 60.199251291408174, + "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 5.97705078125, + "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 7.529541015625, + "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, + "BasicPerformanceMetrics.NUM_FRAMES_SENT": 300, + "BasicPerformanceMetrics.MEAN_FRAME_RATE": 60.180503309413446, + "BasicPerformanceMetrics.MAX_JITTER": 5.021484375, + "BasicPerformanceMetrics.MIN_JITTER": 0.00048828125, + "BasicPerformanceMetrics.MEAN_JITTER": 1.8719007248846478, + "BasicPerformanceMetrics.STD_DEV_JITTER": 1.1710299220677072, + "ResourceMetrics.BASELINE_OVERALL_CPU_UTILIZATION": 3.125, + "ResourceMetrics.MEAN_OVERALL_CPU_UTILIZATION": 1.5424250535331905, + "ResourceMetrics.MAX_OVERALL_CPU_UTILIZATION": 15.625, + "ResourceMetrics.MIN_OVERALL_CPU_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_OVERALL_CPU_UTILIZATION": 2.612885365931027, + "ResourceMetrics.BASELINE_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MAX_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MIN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.BASELINE_DEVICE_MEMORY_UTILIZATION": 1.94, + "ResourceMetrics.MEAN_DEVICE_MEMORY_UTILIZATION": 1.9400000000000004, + "ResourceMetrics.STDDEV_DEVICE_MEMORY_UTILIZATION": 4.440892098500626e-16, + "ResourceMetrics.MAX_DEVICE_MEMORY_UTILIZATION": 1.94, + "ResourceMetrics.MIN_DEVICE_MEMORY_UTILIZATION": 1.94 + }, + "30.0fps": { + "BasicPerformanceMetrics.RECEIVED_DURATION": 4968.640625, + "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 30.201649096555816, + "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 5.700927734375, + "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 7.725341796875, + "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, + "BasicPerformanceMetrics.NUM_FRAMES_SENT": 150, + "BasicPerformanceMetrics.MEAN_FRAME_RATE": 30.189343790586587, + "BasicPerformanceMetrics.MAX_JITTER": 4.933837890625, + "BasicPerformanceMetrics.MIN_JITTER": 0.016845703125, + "BasicPerformanceMetrics.MEAN_JITTER": 1.8972481392525338, + "BasicPerformanceMetrics.STD_DEV_JITTER": 1.181319584209296, + "ResourceMetrics.BASELINE_OVERALL_CPU_UTILIZATION": 3.125, + "ResourceMetrics.MEAN_OVERALL_CPU_UTILIZATION": 0.8460769849785408, + "ResourceMetrics.MAX_OVERALL_CPU_UTILIZATION": 12.5, + "ResourceMetrics.MIN_OVERALL_CPU_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_OVERALL_CPU_UTILIZATION": 2.0431715741839196, + "ResourceMetrics.BASELINE_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MEAN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.STDDEV_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MAX_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.MIN_DEVICE_UTILIZATION": 0.0, + "ResourceMetrics.BASELINE_DEVICE_MEMORY_UTILIZATION": 1.94, + "ResourceMetrics.MEAN_DEVICE_MEMORY_UTILIZATION": 1.9400000000000002, + "ResourceMetrics.STDDEV_DEVICE_MEMORY_UTILIZATION": 2.220446049250313e-16, + "ResourceMetrics.MAX_DEVICE_MEMORY_UTILIZATION": 1.94, + "ResourceMetrics.MIN_DEVICE_MEMORY_UTILIZATION": 1.94 + }, + "custom": { + "data_resolution": "Quarter HD (960,540)" + }, + "metadata": { + "BenchmarkMetadata.NAME": "stereo_image_proc::DisparityNode Benchmark", + "BenchmarkMetadata.TEST_FILE_PATH": "/opt/ros/humble/share/ros2_benchmark/scripts/stereo_image_proc_node.py", + "BenchmarkMetadata.TEST_DATETIME": "2024-12-06T12:33:14Z", + "BenchmarkMetadata.DEVICE_HOSTNAME": "321ae8f3c6dd4a52-0bcd75a369af46cb", + "BenchmarkMetadata.DEVICE_ARCH": "x86_64", + "BenchmarkMetadata.DEVICE_OS": "Linux 6.8.0-48-generic #48~22.04.1-Ubuntu SMP PREEMPT_DYNAMIC Mon Oct 7 11:24:13 UTC 2", + "BenchmarkMetadata.IDLE_CPU_UTIL": 0.03125, + "BenchmarkMetadata.IDLE_GPU_UTIL": 0, + "BenchmarkMetadata.BENCHMARK_MODE": 1, + "BenchmarkMetadata.PEAK_THROUGHPUT_PREDICTION": 99.375, + "BenchmarkMetadata.INPUT_DATA_PATH": "/workspaces/isaac_ros-dev/src/ros2_benchmark/assets/datasets/r2b_dataset/r2b_hideaway", + "BenchmarkMetadata.INPUT_DATA_SIZE": 1913946112, + "BenchmarkMetadata.INPUT_DATA_HASH": "96a3c2b8007173f9da57e848b4fcee8c", + "BenchmarkMetadata.CONFIG": "ros2_benchmark_config:\n additional_fixed_publisher_rate_tests:\n - 10.0\n - 30.0\n - 60.0\n assets_root: /workspaces/isaac_ros-dev/src/ros2_benchmark/assets\n benchmark_duration: 5.0\n benchmark_mode: BenchmarkMode.LOOPING\n benchmark_name: stereo_image_proc::DisparityNode Benchmark\n benchmark_namespace: r2b\n binary_search_acceptable_frame_loss_fraction: 0.05\n binary_search_acceptable_frame_rate_drop: 10.0\n binary_search_duration_fraction: 1.0\n binary_search_terminal_interval_width: 10.0\n collect_start_timestamps_from_monitors: false\n custom_report_info:\n data_resolution: Resolution(960, 540, Quarter HD)\n default_config_version: 0.30.0\n default_service_future_timeout_sec: 15.0\n enable_resource_profiler: true\n enable_trial_buffer_preparation: false\n enforce_publisher_rate: false\n export_monitor_raw_data: false\n input_data_end_time: -1.0\n input_data_path: datasets/r2b_dataset/r2b_hideaway\n input_data_start_time: -1.0\n linear_scan_acceptable_frame_loss_fraction: 0.05\n linear_scan_acceptable_frame_rate_drop: 5.0\n linear_scan_duration_fraction: 1.0\n linear_scan_step_size: 5.0\n load_data_in_real_time: false\n log_file_name: kpi.json\n log_folder: /osmo/data/output\n monitor_info_list:\n - calculators:\n - class_name: BasicPerformanceCalculator\n config:\n report_prefix: ''\n module_name: ros2_benchmark.basic_performance_calculator\n service_name: monitor_node0\n play_messages_service_future_timeout_sec: 15.0\n playback_message_buffer_size: 50\n post_trial_run_wait_time_sec: 0.0\n pre_stop_monitoring_wait_time_sec: 5.0\n pre_trial_run_wait_time_sec: 0.0\n publish_tf_messages_in_set_data: false\n publish_tf_static_messages_in_set_data: false\n publisher_lower_frequency: 10.0\n publisher_upper_frequency: 100.0\n record_data_timeline: false\n resource_profiling_interval_sec: 0.01\n revise_timestamps_as_message_ids: false\n ros1_ws: /workspaces/isaac_ros-dev/isaac_ros_1-dev\n set_data_service_future_timeout_sec: 60.0\n setup_service_client_timeout_sec: 20.0\n start_recording_service_future_timeout_sec: 35.0\n start_recording_service_timeout_sec: 30\n test_iterations: 5\n" + } +} \ No newline at end of file diff --git a/results/stereo_image_proc_node-x86_4090.json b/results/stereo_image_proc_node-x86_4090.json deleted file mode 100644 index 2275a23..0000000 --- a/results/stereo_image_proc_node-x86_4090.json +++ /dev/null @@ -1,90 +0,0 @@ -{ - "BasicPerformanceMetrics.RECEIVED_DURATION": 4985.6220703125, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 77.70055723929134, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 77.095496072621, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 2.6666666666666665, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 387.0, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 15.621175130208334, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 20.818196614583332, - "BasicPerformanceMetrics.MAX_JITTER": 8.25732421875, - "BasicPerformanceMetrics.MIN_JITTER": 0.004150390625, - "BasicPerformanceMetrics.MEAN_JITTER": 2.2826431516617065, - "BasicPerformanceMetrics.STD_DEV_JITTER": 1.6068175167814467, - "CPUProfilingMetrics.MAX_CPU_UTIL": 20.4375, - "CPUProfilingMetrics.MIN_CPU_UTIL": 0.25, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 9.214393939393938, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 9.431482039925118, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 19.58125, - "10.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4898.991455078125, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 10.203771000872852, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 10.20618232517465, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 50, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 16.285888671875, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 15.128173828125, - "BasicPerformanceMetrics.MAX_JITTER": 10.49462890625, - "BasicPerformanceMetrics.MIN_JITTER": 0.1533203125, - "BasicPerformanceMetrics.MEAN_JITTER": 3.900726318359375, - "BasicPerformanceMetrics.STD_DEV_JITTER": 2.9405307170163657, - "CPUProfilingMetrics.MAX_CPU_UTIL": 3.8, - "CPUProfilingMetrics.MIN_CPU_UTIL": 2.93125, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 3.2612499999999995, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 0.31265995906095806, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 3.8 - }, - "60.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4982.83203125, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 60.200929420436225, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 60.206725436165584, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 300, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 14.41943359375, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 13.939697265625, - "BasicPerformanceMetrics.MAX_JITTER": 11.40869140625, - "BasicPerformanceMetrics.MIN_JITTER": 0.001220703125, - "BasicPerformanceMetrics.MEAN_JITTER": 3.731828727978188, - "BasicPerformanceMetrics.STD_DEV_JITTER": 2.5827070058664585, - "CPUProfilingMetrics.MAX_CPU_UTIL": 17.225, - "CPUProfilingMetrics.MIN_CPU_UTIL": 14.993750000000002, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 16.247500000000002, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 0.7970021957309776, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 17.225 - }, - "30.0fps": { - "BasicPerformanceMetrics.RECEIVED_DURATION": 4969.115478515625, - "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 30.2011591857681, - "BasicPerformanceMetrics.MEAN_FRAME_RATE": 30.186458867486017, - "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, - "BasicPerformanceMetrics.NUM_FRAMES_SENT": 150, - "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 12.231201171875, - "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 14.64990234375, - "BasicPerformanceMetrics.MAX_JITTER": 8.697998046875, - "BasicPerformanceMetrics.MIN_JITTER": 0.045166015625, - "BasicPerformanceMetrics.MEAN_JITTER": 3.2797422666807434, - "BasicPerformanceMetrics.STD_DEV_JITTER": 2.1812213447301634, - "CPUProfilingMetrics.MAX_CPU_UTIL": 8.4375, - "CPUProfilingMetrics.MIN_CPU_UTIL": 7.29375, - "CPUProfilingMetrics.MEAN_CPU_UTIL": 7.811250000000001, - "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 0.4223150482755736, - "CPUProfilingMetrics.BASELINE_CPU_UTIL": 8.1125 - }, - "custom": { - "data_resolution": "Quarter HD (960,540)" - }, - "metadata": { - "BenchmarkMetadata.NAME": "stereo_image_proc::DisparityNode Benchmark", - "BenchmarkMetadata.TEST_FILE_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/scripts/stereo_image_proc_node.py", - "BenchmarkMetadata.TEST_DATETIME": "2023-05-25T06:29:10Z", - "BenchmarkMetadata.DEVICE_HOSTNAME": "ros-2n56o6ukyver7iomfrqladfx6u", - "BenchmarkMetadata.DEVICE_ARCH": "x86_64", - "BenchmarkMetadata.DEVICE_OS": "Linux 5.4.0-144-generic #161-Ubuntu SMP Fri Feb 3 14:49:04 UTC 2023", - "BenchmarkMetadata.IDLE_CPU_UTIL": 0.625, - "BenchmarkMetadata.BENCHMARK_MODE": 1, - "BenchmarkMetadata.PEAK_THROUGHPUT_PREDICTION": 77.5, - "BenchmarkMetadata.INPUT_DATA_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets/datasets/r2b_dataset/r2b_hideaway", - "BenchmarkMetadata.INPUT_DATA_SIZE": 1913946112, - "BenchmarkMetadata.INPUT_DATA_HASH": "96a3c2b8007173f9da57e848b4fcee8c", - "BenchmarkMetadata.CONFIG": "ros2_benchmark_config:\n additional_fixed_publisher_rate_tests:\n - 10.0\n - 30.0\n - 60.0\n assets_root: /workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets\n benchmark_duration: 5.0\n benchmark_mode: BenchmarkMode.LOOPING\n benchmark_name: stereo_image_proc::DisparityNode Benchmark\n benchmark_namespace: r2b\n binary_search_acceptable_frame_loss_fraction: 0.05\n binary_search_acceptable_frame_rate_drop: 10.0\n binary_search_duration_fraction: 1.0\n binary_search_terminal_interval_width: 10.0\n cpu_profiling_interval_sec: 1.0\n custom_report_info:\n data_resolution: Resolution(960, 540, Quarter HD)\n default_config_version: 0.30.0\n default_service_future_timeout_sec: 15.0\n enable_cpu_profiler: true\n enable_trial_buffer_preparation: false\n enforce_publisher_rate: false\n input_data_end_time: -1.0\n input_data_path: datasets/r2b_dataset/r2b_hideaway\n input_data_start_time: -1.0\n linear_scan_acceptable_frame_loss_fraction: 0.05\n linear_scan_acceptable_frame_rate_drop: 5.0\n linear_scan_duration_fraction: 1.0\n linear_scan_step_size: 5.0\n load_data_in_real_time: false\n log_file_name: kpi.json\n log_folder: /osmo/data/output\n monitor_info_list:\n - calculators:\n - class_name: BasicPerformanceCalculator\n config:\n report_prefix: ''\n module_name: ros2_benchmark.basic_performance_calculator\n service_name: start_monitoring0\n play_messages_service_future_timeout_sec: 15.0\n playback_message_buffer_size: 50\n publish_tf_messages_in_set_data: false\n publish_tf_static_messages_in_set_data: false\n publisher_lower_frequency: 10.0\n publisher_upper_frequency: 100.0\n record_data_timeline: false\n revise_timestamps_as_message_ids: false\n set_data_service_future_timeout_sec: 60.0\n setup_service_client_timeout_sec: 20.0\n start_monitoring_service_timeout_sec: 10\n start_recording_service_future_timeout_sec: 35.0\n start_recording_service_timeout_sec: 30\n test_iterations: 5\n" - } -} \ No newline at end of file diff --git a/ros2_benchmark/CMakeLists.txt b/ros2_benchmark/CMakeLists.txt index ddb7cc4..ea05db8 100644 --- a/ros2_benchmark/CMakeLists.txt +++ b/ros2_benchmark/CMakeLists.txt @@ -99,4 +99,10 @@ if(BUILD_TESTING) add_launch_test(test/playback_node_pol.py TIMEOUT "60") endif() + +# Embed versioning information into installed files +ament_index_get_resource(ISAAC_ROS_COMMON_CMAKE_PATH isaac_ros_common_cmake_path isaac_ros_common) +include("${ISAAC_ROS_COMMON_CMAKE_PATH}/isaac_ros_common-version-info.cmake") +generate_version_info(${PROJECT_NAME}) + ament_auto_package(INSTALL_TO_SHARE ../scripts) diff --git a/ros2_benchmark/package.xml b/ros2_benchmark/package.xml index 6979cf6..05276c8 100644 --- a/ros2_benchmark/package.xml +++ b/ros2_benchmark/package.xml @@ -21,7 +21,7 @@ SPDX-License-Identifier: Apache-2.0 ros2_benchmark - 3.1.0 + 3.2.0 Benchmarking framework for ROS 2 graphs Isaac ROS Maintainers @@ -32,6 +32,7 @@ SPDX-License-Identifier: Apache-2.0 Yuankun Zhu ament_cmake_auto + isaac_ros_common rclcpp rclcpp_action diff --git a/ros2_benchmark/ros2_benchmark/ros2_benchmark_test.py b/ros2_benchmark/ros2_benchmark/ros2_benchmark_test.py index 9079f43..f213814 100644 --- a/ros2_benchmark/ros2_benchmark/ros2_benchmark_test.py +++ b/ros2_benchmark/ros2_benchmark/ros2_benchmark_test.py @@ -42,11 +42,11 @@ from .basic_performance_calculator import BasicPerformanceMetrics from .ros2_benchmark_config import BenchmarkMode from .ros2_benchmark_config import ROS2BenchmarkConfig -from .utils.cpu_profiler import CPUProfiler from .utils.nsys_utility import NsysUtility from .utils.resource_metrics import ResourceMetrics from .utils.ros2_utility import ClientUtility from .utils.tegrastats_profiler import TegrastatsProfiler +from .utils.x86_profiler import X86Profiler # The maximum allowed line width of a performance repeort displayed in the terminal @@ -121,8 +121,8 @@ def __init__(self, *args, **kwargs): self._resource_profiler = TegrastatsProfiler() rclpy.logging.get_logger('r2b').info('Use tegrastats resource profiler') except FileNotFoundError: - self._resource_profiler = CPUProfiler() - rclpy.logging.get_logger('r2b').info('Use CPU profiler') + self._resource_profiler = X86Profiler() + rclpy.logging.get_logger('r2b').info('Use x86 resource profiler') super().__init__(*args, **kwargs) @@ -186,8 +186,8 @@ def set_idle_resource_utilization(): resource_profiler = TegrastatsProfiler() rclpy.logging.get_logger('r2b').info('Probing with tegrastats resource profiler...') except FileNotFoundError: - resource_profiler = CPUProfiler() - rclpy.logging.get_logger('r2b').info('Probing with CPU profiler...') + resource_profiler = X86Profiler() + rclpy.logging.get_logger('r2b').info('Probing with x86 profiler...') current_usage = resource_profiler.get_current_usage() if ResourceMetrics.MEAN_OVERALL_CPU_UTILIZATION in current_usage: diff --git a/ros2_benchmark/ros2_benchmark/utils/cpu_profiler.py b/ros2_benchmark/ros2_benchmark/utils/cpu_profiler.py deleted file mode 100644 index f35b317..0000000 --- a/ros2_benchmark/ros2_benchmark/utils/cpu_profiler.py +++ /dev/null @@ -1,147 +0,0 @@ -# SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -# Copyright (c) 2021-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# SPDX-License-Identifier: Apache-2.0 - -"""CPU profiler class to measure performance of benchmark tests.""" - -import numbers -from pathlib import Path -from threading import Thread - -import numpy as np -import psutil - -from .profiler import Profiler -from .resource_metrics import ResourceMetrics - - -class CPUProfiler(Profiler): - """CPU profiler class to measure CPU performance of benchmark tests.""" - - def __init__(self): - """Construct CPU profiler.""" - super().__init__() - - def start_profiling(self, interval: float = 1.0) -> Path: - """ - Start CPU profiling thread to keep track of performance metrics. - - Parameters - ---------- - interval: float - The interval between measurements, in seconds - - """ - super().start_profiling() - - # While the is_running flag is true, log CPU usage - def psutil_log(): - with open(self._log_file_path, 'w+') as logfile: - while self._is_running: - logfile.write( - f'{psutil.cpu_percent(interval=interval, percpu=True)}\n') - - self.psutil_thread = Thread(target=psutil_log) - self.psutil_thread.start() - - return self._log_file_path - - def stop_profiling(self): - """Stop profiling.""" - if self._is_running: - super().stop_profiling() - # Wait for thread to stop - self.psutil_thread.join() - - @staticmethod - def get_current_usage(): - """Return current CPU usage.""" - profile_data = {} - profile_data[ResourceMetrics.MEAN_OVERALL_CPU_UTILIZATION] = \ - np.mean(psutil.cpu_percent(interval=1.0, percpu=True)) - return profile_data - - def get_results(self, log_file_path=None) -> dict: - """Return CPU profiling results.""" - assert not self._is_running, 'Cannot collect results until profiler has been stopped!' - - log_file_path = self._log_file_path if log_file_path is None else log_file_path - assert self._log_file_path is not None, 'No log file for reading CPU profiling results.' - - profile_data = {} - with open(log_file_path) as logfile: - cpu_values = [] - for line in logfile.readlines(): - # Remove brackets from line before splitting entries by comma - cpu_values.append(np.mean([float(v) - for v in line[1:-2].split(',')])) - - cpu_values = np.array(cpu_values) - profile_data[ResourceMetrics.BASELINE_OVERALL_CPU_UTILIZATION] = cpu_values[0] - profile_data[ResourceMetrics.MAX_OVERALL_CPU_UTILIZATION] = np.max(cpu_values) - profile_data[ResourceMetrics.MIN_OVERALL_CPU_UTILIZATION] = np.min(cpu_values) - profile_data[ResourceMetrics.MEAN_OVERALL_CPU_UTILIZATION] = np.mean(cpu_values) - profile_data[ResourceMetrics.STDDEV_OVERALL_CPU_UTILIZATION] = np.std(cpu_values) - - self._profile_data_list.append(profile_data) - - return profile_data - - def reset(self): - """Reset the profiler state.""" - self._profile_data_list.clear() - return - - def conclude_results(self) -> dict: - """Conclude final profiling outcome based on all previous results.""" - if len(self._profile_data_list) == 0: - self.get_logger().warn('No prior profile data to conclude') - return {} - - MEAN_METRICS = [ - ResourceMetrics.BASELINE_OVERALL_CPU_UTILIZATION, - ResourceMetrics.MEAN_OVERALL_CPU_UTILIZATION, - ResourceMetrics.STDDEV_OVERALL_CPU_UTILIZATION, - ] - MAX_METRICS = [ - ResourceMetrics.MAX_OVERALL_CPU_UTILIZATION, - ] - MIN_METRICS = [ - ResourceMetrics.MIN_OVERALL_CPU_UTILIZATION, - ] - - final_profile_data = {} - for metric in ResourceMetrics: - metric_value_list = [profile_data.get(metric, None) for - profile_data in self._profile_data_list] - if not all(isinstance(value, numbers.Number) for value in metric_value_list): - continue - - # Remove the best and the worst before concluding the metric - metric_value_list.remove(max(metric_value_list)) - metric_value_list.remove(min(metric_value_list)) - - if metric in MEAN_METRICS: - final_profile_data[metric] = sum(metric_value_list)/len(metric_value_list) - elif metric in MAX_METRICS: - final_profile_data[metric] = max(metric_value_list) - elif metric in MIN_METRICS: - final_profile_data[metric] = min(metric_value_list) - else: - final_profile_data[metric] = 'INVALID VALUES: NO CONCLUDED METHOD ASSIGNED' - - self.reset() - return final_profile_data diff --git a/ros2_benchmark/ros2_benchmark/utils/nsys_utility.py b/ros2_benchmark/ros2_benchmark/utils/nsys_utility.py index 3649973..b077bb4 100644 --- a/ros2_benchmark/ros2_benchmark/utils/nsys_utility.py +++ b/ros2_benchmark/ros2_benchmark/utils/nsys_utility.py @@ -58,7 +58,7 @@ def generate_nsys_prefix(context): container_prefix = '' if enable_nsys: - if(not nsys_profile_name): + if not nsys_profile_name: current_time = datetime.datetime.now(datetime.timezone.utc).\ strftime('%Y-%m-%dT%H:%M:%SZ') nsys_profile_name = f'profile_{platform.machine()}_{current_time}' diff --git a/ros2_benchmark/ros2_benchmark/utils/tegrastats_profiler.py b/ros2_benchmark/ros2_benchmark/utils/tegrastats_profiler.py index bc04ce5..9717a8a 100644 --- a/ros2_benchmark/ros2_benchmark/utils/tegrastats_profiler.py +++ b/ros2_benchmark/ros2_benchmark/utils/tegrastats_profiler.py @@ -34,7 +34,7 @@ class TegrastatsProfiler(Profiler): """Tegrastats profiler class to measure CPU and GPU performance of benchmark tests.""" def __init__(self, tegrastats_path='tegrastats'): - """Construct CPU profiler.""" + """Construct Tegrastats profiler.""" super().__init__() self.tegrastats_path = tegrastats_path if shutil.which(self.tegrastats_path) is None: diff --git a/ros2_benchmark/ros2_benchmark/utils/x86_profiler.py b/ros2_benchmark/ros2_benchmark/utils/x86_profiler.py new file mode 100644 index 0000000..ceb20ea --- /dev/null +++ b/ros2_benchmark/ros2_benchmark/utils/x86_profiler.py @@ -0,0 +1,217 @@ +# SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES +# Copyright (c) 2021-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +"""CPU profiler class to measure performance of benchmark tests.""" + +import numbers +from pathlib import Path +from threading import Thread + +import numpy as np +import psutil + +from .profiler import Profiler +from .resource_metrics import ResourceMetrics + +try: + import gpustat + gpustat_imported = True +except Exception: + gpustat_imported = False + +CPU_IDENTIFIER_STR = 'CPU: ' +GPU_IDENTIFIER_STR = 'GPU: ' + + +class X86Profiler(Profiler): + """x86 profiler class to measure x86 performance of benchmark tests.""" + + def __init__(self): + """Construct x86 profiler.""" + super().__init__() + if not gpustat_imported: + self.get_logger().warn('Failed to import gpustat. Disabled GPU profiling.') + + def start_profiling(self, interval: float = 1.0) -> Path: + """ + Start x86 profiling thread to keep track of performance metrics. + + Parameters + ---------- + interval: float + The interval between measurements, in seconds + + """ + super().start_profiling() + + # While the is_running flag is true, log CPU/GPU usage + def profiler_log_func(): + with open(self._log_file_path, 'w+') as logfile: + while self._is_running: + # Measure CPU stats + cpu_line = f'{CPU_IDENTIFIER_STR}' \ + f'{psutil.cpu_percent(interval=interval, percpu=True)}\n' + logfile.write(cpu_line) + + if gpustat_imported: + # Measure GPU stats + gpu_query = gpustat.new_query().gpus[0] + gpu_util = gpu_query.utilization + gpu_memory_util = round( + 100 * gpu_query.memory_used / gpu_query.memory_total, 2) + gpu_line = f'{GPU_IDENTIFIER_STR}{gpu_util},{gpu_memory_util}\n' + logfile.write(gpu_line) + + self.profiler_thread = Thread(target=profiler_log_func) + self.profiler_thread.start() + + return self._log_file_path + + def stop_profiling(self): + """Stop profiling.""" + if self._is_running: + super().stop_profiling() + # Wait for thread to stop + self.profiler_thread.join() + + @staticmethod + def get_current_usage(): + """Return current x86 usage.""" + profile_data = {} + profile_data[ResourceMetrics.MEAN_OVERALL_CPU_UTILIZATION] = \ + np.mean(psutil.cpu_percent(interval=1.0, percpu=True)) + if gpustat_imported: + gpu_query = gpustat.new_query().gpus[0] + profile_data[ResourceMetrics.MEAN_DEVICE_UTILIZATION] = \ + gpu_query.utilization + profile_data[ResourceMetrics.MEAN_DEVICE_MEMORY_UTILIZATION] = \ + round(100 * gpu_query.memory_used / gpu_query.memory_total, 2) + return profile_data + + def get_results(self, log_file_path=None) -> dict: + """Return x86 profiling results.""" + assert not self._is_running, 'Cannot collect results until profiler has been stopped!' + + log_file_path = self._log_file_path if log_file_path is None else log_file_path + assert self._log_file_path is not None, 'No log file for reading CPU profiling results.' + + profile_data = {} + with open(log_file_path) as logfile: + cpu_util_values = [] + gpu_util_values = [] + gpu_memory_values = [] + for line in logfile.readlines(): + # Remove brackets from line before splitting entries by comma + + if line[:len(CPU_IDENTIFIER_STR)] == CPU_IDENTIFIER_STR: + line = line[len(CPU_IDENTIFIER_STR):] + cpu_util_values.append(np.mean([float(v) for v in line[1:-2].split(',')])) + + if line[:len(GPU_IDENTIFIER_STR)] == GPU_IDENTIFIER_STR: + line = line[len(GPU_IDENTIFIER_STR):] + gpu_value_list = line.replace('\n', '').split(',') + gpu_util_values.append(float(gpu_value_list[0])) + gpu_memory_values.append(float(gpu_value_list[1])) + + cpu_util_values = np.array(cpu_util_values) + gpu_util_values = np.array(gpu_util_values) + gpu_memory_values = np.array(gpu_memory_values) + + profile_data[ResourceMetrics.BASELINE_OVERALL_CPU_UTILIZATION] = cpu_util_values[0] + profile_data[ResourceMetrics.MEAN_OVERALL_CPU_UTILIZATION] = np.mean(cpu_util_values) + profile_data[ResourceMetrics.MAX_OVERALL_CPU_UTILIZATION] = np.max(cpu_util_values) + profile_data[ResourceMetrics.MIN_OVERALL_CPU_UTILIZATION] = np.min(cpu_util_values) + profile_data[ResourceMetrics.STDDEV_OVERALL_CPU_UTILIZATION] = np.std(cpu_util_values) + + if len(gpu_util_values) > 0: + profile_data[ResourceMetrics.BASELINE_DEVICE_UTILIZATION] = gpu_util_values[0] + profile_data[ResourceMetrics.MEAN_DEVICE_UTILIZATION] = np.mean(gpu_util_values) + profile_data[ResourceMetrics.STDDEV_DEVICE_UTILIZATION] = np.std(gpu_util_values) + profile_data[ResourceMetrics.MAX_DEVICE_UTILIZATION] = max(gpu_util_values) + profile_data[ResourceMetrics.MIN_DEVICE_UTILIZATION] = min(gpu_util_values) + + if len(gpu_memory_values) > 0: + profile_data[ResourceMetrics.BASELINE_DEVICE_MEMORY_UTILIZATION] = \ + gpu_memory_values[0] + profile_data[ResourceMetrics.MEAN_DEVICE_MEMORY_UTILIZATION] = np.mean( + gpu_memory_values) + profile_data[ResourceMetrics.STDDEV_DEVICE_MEMORY_UTILIZATION] = np.std( + gpu_memory_values) + profile_data[ResourceMetrics.MAX_DEVICE_MEMORY_UTILIZATION] = \ + max(gpu_memory_values) + profile_data[ResourceMetrics.MIN_DEVICE_MEMORY_UTILIZATION] = \ + min(gpu_memory_values) + + self._profile_data_list.append(profile_data) + + return profile_data + + def reset(self): + """Reset the profiler state.""" + self._profile_data_list.clear() + return + + def conclude_results(self) -> dict: + """Conclude final profiling outcome based on all previous results.""" + if len(self._profile_data_list) == 0: + self.get_logger().warn('No prior profile data to conclude') + return {} + + MEAN_METRICS = [ + ResourceMetrics.BASELINE_OVERALL_CPU_UTILIZATION, + ResourceMetrics.BASELINE_DEVICE_UTILIZATION, + ResourceMetrics.BASELINE_DEVICE_MEMORY_UTILIZATION, + ResourceMetrics.MEAN_OVERALL_CPU_UTILIZATION, + ResourceMetrics.MEAN_DEVICE_UTILIZATION, + ResourceMetrics.MEAN_DEVICE_MEMORY_UTILIZATION, + ResourceMetrics.STDDEV_OVERALL_CPU_UTILIZATION, + ResourceMetrics.STDDEV_DEVICE_UTILIZATION, + ResourceMetrics.STDDEV_DEVICE_MEMORY_UTILIZATION, + ] + MAX_METRICS = [ + ResourceMetrics.MAX_OVERALL_CPU_UTILIZATION, + ResourceMetrics.MAX_DEVICE_UTILIZATION, + ResourceMetrics.MAX_DEVICE_MEMORY_UTILIZATION, + ] + MIN_METRICS = [ + ResourceMetrics.MIN_OVERALL_CPU_UTILIZATION, + ResourceMetrics.MIN_DEVICE_UTILIZATION, + ResourceMetrics.MIN_DEVICE_MEMORY_UTILIZATION, + ] + + final_profile_data = {} + for metric in ResourceMetrics: + metric_value_list = [ + profile_data.get(metric, None) for profile_data in self._profile_data_list + ] + if not all(isinstance(value, numbers.Number) for value in metric_value_list): + continue + + # Remove the best and the worst before concluding the metric + metric_value_list.remove(max(metric_value_list)) + metric_value_list.remove(min(metric_value_list)) + + if metric in MEAN_METRICS: + final_profile_data[metric] = sum(metric_value_list) / len(metric_value_list) + elif metric in MAX_METRICS: + final_profile_data[metric] = max(metric_value_list) + elif metric in MIN_METRICS: + final_profile_data[metric] = min(metric_value_list) + else: + final_profile_data[metric] = 'INVALID VALUES: NO CONCLUDED METHOD ASSIGNED' + + self.reset() + return final_profile_data diff --git a/ros2_benchmark_interfaces/CMakeLists.txt b/ros2_benchmark_interfaces/CMakeLists.txt index ebf20ee..5865c8d 100644 --- a/ros2_benchmark_interfaces/CMakeLists.txt +++ b/ros2_benchmark_interfaces/CMakeLists.txt @@ -67,4 +67,10 @@ if(BUILD_TESTING) endif() + +# Embed versioning information into installed files +ament_index_get_resource(ISAAC_ROS_COMMON_CMAKE_PATH isaac_ros_common_cmake_path isaac_ros_common) +include("${ISAAC_ROS_COMMON_CMAKE_PATH}/isaac_ros_common-version-info.cmake") +generate_version_info(${PROJECT_NAME}) + ament_auto_package() diff --git a/ros2_benchmark_interfaces/package.xml b/ros2_benchmark_interfaces/package.xml index 62058fb..7b5c447 100644 --- a/ros2_benchmark_interfaces/package.xml +++ b/ros2_benchmark_interfaces/package.xml @@ -21,7 +21,7 @@ SPDX-License-Identifier: Apache-2.0 ros2_benchmark_interfaces - 3.1.0 + 3.2.0 Interfaces for benchmark testing Isaac ROS Maintainers @@ -32,6 +32,7 @@ SPDX-License-Identifier: Apache-2.0 Yuankun Zhu rosidl_default_generators + isaac_ros_common rosidl_default_runtime ament_lint_auto