diff --git a/qai_hub_models/models/aotgan/export.py b/qai_hub_models/models/aotgan/export.py index 224f1c73..e86faccd 100644 --- a/qai_hub_models/models/aotgan/export.py +++ b/qai_hub_models/models/aotgan/export.py @@ -187,11 +187,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/aotgan/perf.yaml b/qai_hub_models/models/aotgan/perf.yaml index 23be94b6..db1d414b 100644 --- a/qai_hub_models/models/aotgan/perf.yaml +++ b/qai_hub_models/models/aotgan/perf.yaml @@ -65,21 +65,6 @@ models: total_layers: 275 job_id: jwgovdz45 job_status: Passed - torchscript_onnx_ort: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: jygz7z2zp - job_status: Failed reference_device_info: name: Samsung Galaxy S23 os: '13' @@ -87,7 +72,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:49.396717Z' + timestamp: '2024-05-29T18:59:39.212193Z' - torchscript_onnx_tflite: inference_time: 120767.0 throughput: 8.280407727276492 @@ -118,21 +103,6 @@ models: total_layers: 275 job_id: j1pvw2q7g job_status: Passed - torchscript_onnx_ort: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: jz5w9ywzp - job_status: Failed reference_device_info: name: Samsung Galaxy S24 os: '14' @@ -140,7 +110,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:49.517726Z' + timestamp: '2024-05-29T18:59:39.339992Z' - torchscript_onnx_tflite: inference_time: 164352.0 throughput: 6.084501557632398 @@ -178,7 +148,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:49.635903Z' + timestamp: '2024-05-29T18:59:39.469404Z' - torchscript_onnx_qnn: inference_time: 145505.0 throughput: 6.872616061303735 @@ -194,36 +164,6 @@ models: total_layers: 275 job_id: j7gjl3d7p job_status: Passed - torchscript_onnx_ort: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: jmg94o0q5 - job_status: Failed - torchscript_onnx_ort_dml_gpu: - inference_time: 739119.0 - throughput: 1.3529621075902527 - estimated_peak_memory_range: - min: 416927744 - max: 416927744 - primary_compute_unit: CPU - precision: fp32 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 234 - total_layers: 234 - job_id: jnp18o2kg - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -231,4 +171,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:49.749014Z' + timestamp: '2024-05-29T18:59:39.597982Z' diff --git a/qai_hub_models/models/convnext_tiny/export.py b/qai_hub_models/models/convnext_tiny/export.py index c8978ee9..7342e877 100644 --- a/qai_hub_models/models/convnext_tiny/export.py +++ b/qai_hub_models/models/convnext_tiny/export.py @@ -185,11 +185,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/convnext_tiny/perf.yaml b/qai_hub_models/models/convnext_tiny/perf.yaml index 2a20afdf..80ea16f7 100644 --- a/qai_hub_models/models/convnext_tiny/perf.yaml +++ b/qai_hub_models/models/convnext_tiny/perf.yaml @@ -87,7 +87,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:49.806533Z' + timestamp: '2024-05-29T18:59:39.628526Z' - torchscript_onnx_tflite: inference_time: 3967.0 throughput: 252.07965717166624 @@ -140,7 +140,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:49.806634Z' + timestamp: '2024-05-29T18:59:39.628637Z' - torchscript_onnx_tflite: inference_time: 5705.0 throughput: 175.28483786152498 @@ -178,7 +178,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:49.806715Z' + timestamp: '2024-05-29T18:59:39.628718Z' - torchscript_onnx_qnn: inference_time: 3927.0 throughput: 254.64731347084287 @@ -209,21 +209,6 @@ models: total_layers: 189 job_id: j2p0ro32p job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 35785.0 - throughput: 27.94466955428252 - estimated_peak_memory_range: - min: 1454080 - max: 1454080 - primary_compute_unit: CPU - precision: fp32 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 202 - total_layers: 202 - job_id: j1p87j0z5 - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -231,4 +216,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:49.806795Z' + timestamp: '2024-05-29T18:59:39.628784Z' diff --git a/qai_hub_models/models/convnext_tiny_w8a16_quantized/export.py b/qai_hub_models/models/convnext_tiny_w8a16_quantized/export.py index c1fb2182..beeafaae 100644 --- a/qai_hub_models/models/convnext_tiny_w8a16_quantized/export.py +++ b/qai_hub_models/models/convnext_tiny_w8a16_quantized/export.py @@ -193,11 +193,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/convnext_tiny_w8a16_quantized/perf.yaml b/qai_hub_models/models/convnext_tiny_w8a16_quantized/perf.yaml index 8aa99742..0ae7d848 100644 --- a/qai_hub_models/models/convnext_tiny_w8a16_quantized/perf.yaml +++ b/qai_hub_models/models/convnext_tiny_w8a16_quantized/perf.yaml @@ -63,7 +63,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:50.094158Z' + timestamp: '2024-05-29T18:59:39.928338Z' - torchscript_onnx_qnn: inference_time: 2291.0 throughput: 436.4906154517678 @@ -86,7 +86,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:50.094352Z' + timestamp: '2024-05-29T18:59:39.928544Z' - torchscript_onnx_qnn: inference_time: 3277.0 throughput: 305.15715593530666 @@ -109,7 +109,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:50.094519Z' + timestamp: '2024-05-29T18:59:39.928727Z' - torchscript_onnx_qnn: inference_time: 11165.0 throughput: 89.56560680698611 @@ -132,7 +132,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs6490 - timestamp: '2024-05-24T06:12:50.216033Z' + timestamp: '2024-05-29T18:59:40.073528Z' - reference_device_info: name: RB5 (Proxy) os: '12' @@ -140,7 +140,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8250 - timestamp: '2024-05-24T06:12:50.334252Z' + timestamp: '2024-05-29T18:59:40.202310Z' - torchscript_onnx_qnn: inference_time: 3621.0 throughput: 276.16680475006905 @@ -163,4 +163,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:50.334505Z' + timestamp: '2024-05-29T18:59:40.203221Z' diff --git a/qai_hub_models/models/convnext_tiny_w8a8_quantized/export.py b/qai_hub_models/models/convnext_tiny_w8a8_quantized/export.py index 0c2c9725..787f0f4f 100644 --- a/qai_hub_models/models/convnext_tiny_w8a8_quantized/export.py +++ b/qai_hub_models/models/convnext_tiny_w8a8_quantized/export.py @@ -193,11 +193,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/convnext_tiny_w8a8_quantized/perf.yaml b/qai_hub_models/models/convnext_tiny_w8a8_quantized/perf.yaml index fdcadac7..08bb11f7 100644 --- a/qai_hub_models/models/convnext_tiny_w8a8_quantized/perf.yaml +++ b/qai_hub_models/models/convnext_tiny_w8a8_quantized/perf.yaml @@ -41,22 +41,7 @@ aggregated: models: - name: ConvNext-Tiny-w8a8-Quantized performance_metrics: - - torchscript_onnx_tflite: - inference_time: 33794.0 - throughput: 29.59105166597621 - estimated_peak_memory_range: - min: 6995968 - max: 35792496 - primary_compute_unit: NPU - precision: int8 - layer_info: - layers_on_npu: 809 - layers_on_gpu: 2 - layers_on_cpu: 34 - total_layers: 845 - job_id: jlpev6e75 - job_status: Passed - torchscript_onnx_qnn: + - torchscript_onnx_qnn: inference_time: 1732.0 throughput: 577.3672055427252 estimated_peak_memory_range: @@ -71,21 +56,6 @@ models: total_layers: 215 job_id: jvgdv6ekg job_status: Passed - torchscript_onnx_ort: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: jegne1mvg - job_status: Failed reference_device_info: name: Samsung Galaxy S23 os: '13' @@ -93,23 +63,8 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:50.345058Z' - - torchscript_onnx_tflite: - inference_time: 30704.0 - throughput: 32.56904637832204 - estimated_peak_memory_range: - min: 5738496 - max: 168067456 - primary_compute_unit: NPU - precision: int8 - layer_info: - layers_on_npu: 809 - layers_on_gpu: 2 - layers_on_cpu: 34 - total_layers: 845 - job_id: jygz7zozp - job_status: Passed - torchscript_onnx_qnn: + timestamp: '2024-05-29T18:59:40.231037Z' + - torchscript_onnx_qnn: inference_time: 1204.0 throughput: 830.5647840531561 estimated_peak_memory_range: @@ -124,21 +79,6 @@ models: total_layers: 215 job_id: jz57do0q5 job_status: Passed - torchscript_onnx_ort: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: jopryx2vg - job_status: Failed reference_device_info: name: Samsung Galaxy S24 os: '14' @@ -146,23 +86,8 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:50.345188Z' - - torchscript_onnx_tflite: - inference_time: 33981.0 - throughput: 29.428209881992878 - estimated_peak_memory_range: - min: 6955008 - max: 67832576 - primary_compute_unit: NPU - precision: int8 - layer_info: - layers_on_npu: 809 - layers_on_gpu: 2 - layers_on_cpu: 34 - total_layers: 845 - job_id: jz5w9y2zp - job_status: Passed - torchscript_onnx_qnn: + timestamp: '2024-05-29T18:59:40.231081Z' + - torchscript_onnx_qnn: inference_time: 1729.0 throughput: 578.368999421631 estimated_peak_memory_range: @@ -184,23 +109,8 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:50.345304Z' - - torchscript_onnx_tflite: - inference_time: 146562.0 - throughput: 6.823050995483141 - estimated_peak_memory_range: - min: 23556096 - max: 93702336 - primary_compute_unit: NPU - precision: int8 - layer_info: - layers_on_npu: 755 - layers_on_gpu: 3 - layers_on_cpu: 87 - total_layers: 845 - job_id: jmg94ojq5 - job_status: Passed - torchscript_onnx_qnn: + timestamp: '2024-05-29T18:59:40.231120Z' + - torchscript_onnx_qnn: inference_time: 6345.0 throughput: 157.60441292356185 estimated_peak_memory_range: @@ -222,30 +132,15 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs6490 - timestamp: '2024-05-24T06:12:50.345416Z' - - torchscript_onnx_tflite: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: jnp18oykg - job_status: Failed - reference_device_info: + timestamp: '2024-05-29T18:59:40.231157Z' + - reference_device_info: name: RB5 (Proxy) os: '12' form_factor: Iot os_name: Android manufacturer: Qualcomm chipset: Qcs8250 - timestamp: '2024-05-24T06:12:50.345427Z' + timestamp: '2024-05-29T18:59:40.231162Z' - torchscript_onnx_qnn: inference_time: 1931.0 throughput: 517.8663904712585 @@ -261,36 +156,6 @@ models: total_layers: 215 job_id: jqp4wekqg job_status: Passed - torchscript_onnx_ort: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: jep2mo9x5 - job_status: Failed - torchscript_onnx_ort_dml_gpu: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: jqpyd8jrp - job_status: Failed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -298,4 +163,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:50.345471Z' + timestamp: '2024-05-29T18:59:40.231201Z' diff --git a/qai_hub_models/models/ddrnet23_slim/export.py b/qai_hub_models/models/ddrnet23_slim/export.py index 5b9e6edc..8ed00b91 100644 --- a/qai_hub_models/models/ddrnet23_slim/export.py +++ b/qai_hub_models/models/ddrnet23_slim/export.py @@ -184,11 +184,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/ddrnet23_slim/perf.yaml b/qai_hub_models/models/ddrnet23_slim/perf.yaml index c1eb7d98..62939a16 100644 --- a/qai_hub_models/models/ddrnet23_slim/perf.yaml +++ b/qai_hub_models/models/ddrnet23_slim/perf.yaml @@ -72,7 +72,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:50.358544Z' + timestamp: '2024-05-29T18:59:40.256306Z' - torchscript_onnx_tflite: inference_time: 4767.0 throughput: 209.77554017201595 @@ -110,7 +110,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:50.358595Z' + timestamp: '2024-05-29T18:59:40.256364Z' - torchscript_onnx_tflite: inference_time: 6723.0 throughput: 148.74312063067083 @@ -133,7 +133,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:50.358621Z' + timestamp: '2024-05-29T18:59:40.256392Z' - torchscript_onnx_ort: inference_time: 9545.0 throughput: 104.76689366160294 @@ -149,21 +149,6 @@ models: total_layers: 155 job_id: jw561ozvp job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 38051.0 - throughput: 26.280518251819927 - estimated_peak_memory_range: - min: 104112128 - max: 104112128 - primary_compute_unit: CPU - precision: fp32 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 105 - total_layers: 105 - job_id: j1p3mo1xg - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -171,4 +156,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:50.358668Z' + timestamp: '2024-05-29T18:59:40.256424Z' diff --git a/qai_hub_models/models/deeplabv3_plus_mobilenet/export.py b/qai_hub_models/models/deeplabv3_plus_mobilenet/export.py index 4bba609d..08966976 100644 --- a/qai_hub_models/models/deeplabv3_plus_mobilenet/export.py +++ b/qai_hub_models/models/deeplabv3_plus_mobilenet/export.py @@ -184,11 +184,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/deeplabv3_plus_mobilenet/perf.yaml b/qai_hub_models/models/deeplabv3_plus_mobilenet/perf.yaml index d55d1cbd..43963ab9 100644 --- a/qai_hub_models/models/deeplabv3_plus_mobilenet/perf.yaml +++ b/qai_hub_models/models/deeplabv3_plus_mobilenet/perf.yaml @@ -87,7 +87,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:50.390418Z' + timestamp: '2024-05-29T18:59:40.282247Z' - torchscript_onnx_tflite: inference_time: 9834.0 throughput: 101.6880211511084 @@ -140,7 +140,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:50.390486Z' + timestamp: '2024-05-29T18:59:40.282320Z' - torchscript_onnx_tflite: inference_time: 13172.0 throughput: 75.91861524445794 @@ -178,7 +178,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:50.390527Z' + timestamp: '2024-05-29T18:59:40.282366Z' - torchscript_onnx_qnn: inference_time: 16505.0 throughput: 60.58770069675856 @@ -209,21 +209,6 @@ models: total_layers: 126 job_id: jz5w9y3jp job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 110686.0 - throughput: 9.034566250474315 - estimated_peak_memory_range: - min: 104878080 - max: 104878080 - primary_compute_unit: CPU - precision: fp32 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 69 - total_layers: 69 - job_id: jmg94oyv5 - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -231,4 +216,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:50.390599Z' + timestamp: '2024-05-29T18:59:40.282417Z' diff --git a/qai_hub_models/models/deeplabv3_plus_mobilenet_quantized/export.py b/qai_hub_models/models/deeplabv3_plus_mobilenet_quantized/export.py index a9b69e6e..95facb42 100644 --- a/qai_hub_models/models/deeplabv3_plus_mobilenet_quantized/export.py +++ b/qai_hub_models/models/deeplabv3_plus_mobilenet_quantized/export.py @@ -192,11 +192,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/deeplabv3_plus_mobilenet_quantized/perf.yaml b/qai_hub_models/models/deeplabv3_plus_mobilenet_quantized/perf.yaml index 17a274f2..76310879 100644 --- a/qai_hub_models/models/deeplabv3_plus_mobilenet_quantized/perf.yaml +++ b/qai_hub_models/models/deeplabv3_plus_mobilenet_quantized/perf.yaml @@ -93,7 +93,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:50.421466Z' + timestamp: '2024-05-29T18:59:40.316733Z' - torchscript_onnx_tflite: inference_time: 2493.0 throughput: 401.1231448054553 @@ -146,7 +146,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:50.421534Z' + timestamp: '2024-05-29T18:59:40.316808Z' - torchscript_onnx_tflite: inference_time: 3344.0 throughput: 299.0430622009569 @@ -184,7 +184,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:50.421576Z' + timestamp: '2024-05-29T18:59:40.316853Z' - torchscript_onnx_tflite: inference_time: 15002.0 throughput: 66.65777896280495 @@ -222,7 +222,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs6490 - timestamp: '2024-05-24T06:12:50.421631Z' + timestamp: '2024-05-29T18:59:40.316896Z' - torchscript_onnx_tflite: inference_time: 120249.0 throughput: 8.316077472577735 @@ -245,7 +245,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8250 - timestamp: '2024-05-24T06:12:50.421658Z' + timestamp: '2024-05-29T18:59:40.316921Z' - torchscript_onnx_qnn: inference_time: 5376.0 throughput: 186.01190476190476 @@ -276,21 +276,6 @@ models: total_layers: 173 job_id: jogky63op job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 477287.0 - throughput: 2.095175439515428 - estimated_peak_memory_range: - min: 248303616 - max: 248303616 - primary_compute_unit: GPU - precision: int8 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 1 - layers_on_cpu: 0 - total_layers: 1 - job_id: jn5q243m5 - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -298,4 +283,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:50.421710Z' + timestamp: '2024-05-29T18:59:40.316975Z' diff --git a/qai_hub_models/models/deeplabv3_resnet50/export.py b/qai_hub_models/models/deeplabv3_resnet50/export.py index 62ef691e..b73993c7 100644 --- a/qai_hub_models/models/deeplabv3_resnet50/export.py +++ b/qai_hub_models/models/deeplabv3_resnet50/export.py @@ -185,11 +185,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/deeplabv3_resnet50/perf.yaml b/qai_hub_models/models/deeplabv3_resnet50/perf.yaml index 466ec99e..5f4591d3 100644 --- a/qai_hub_models/models/deeplabv3_resnet50/perf.yaml +++ b/qai_hub_models/models/deeplabv3_resnet50/perf.yaml @@ -50,36 +50,6 @@ models: total_layers: 95 job_id: j1glkw3lp job_status: Passed - torchscript_onnx_qnn: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: jwgovd3d5 - job_status: Failed - torchscript_onnx_ort: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: jlpev6k05 - job_status: Failed reference_device_info: name: Samsung Galaxy S23 os: '13' @@ -87,7 +57,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:50.461216Z' + timestamp: '2024-05-29T18:59:40.360596Z' - torchscript_onnx_tflite: inference_time: 217580.0 throughput: 4.596010662744738 @@ -103,36 +73,6 @@ models: total_layers: 95 job_id: jw561on7p job_status: Passed - torchscript_onnx_qnn: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: j1pvw2vmg - job_status: Failed - torchscript_onnx_ort: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: jygz7zr6p - job_status: Failed reference_device_info: name: Samsung Galaxy S24 os: '14' @@ -140,7 +80,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:50.461255Z' + timestamp: '2024-05-29T18:59:40.360628Z' - torchscript_onnx_tflite: inference_time: 293257.0 throughput: 3.4099782784383663 @@ -156,21 +96,6 @@ models: total_layers: 95 job_id: j1p3moezg job_status: Passed - torchscript_onnx_qnn: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: j7gjl3e8p - job_status: Failed reference_device_info: name: QCS8550 (Proxy) os: '12' @@ -178,42 +103,12 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:50.461283Z' - - torchscript_onnx_ort: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: jz5w9yqjp - job_status: Failed - torchscript_onnx_ort_dml_gpu: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: jmg94owv5 - job_status: Failed - reference_device_info: + timestamp: '2024-05-29T18:59:40.360656Z' + - reference_device_info: name: Snapdragon X Elite CRD os: '11' form_factor: Compute os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:50.461302Z' + timestamp: '2024-05-29T18:59:40.360664Z' diff --git a/qai_hub_models/models/densenet121/export.py b/qai_hub_models/models/densenet121/export.py index 748dc2ae..9e6eb456 100644 --- a/qai_hub_models/models/densenet121/export.py +++ b/qai_hub_models/models/densenet121/export.py @@ -185,11 +185,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/densenet121/perf.yaml b/qai_hub_models/models/densenet121/perf.yaml index c0af15bf..3b6037e6 100644 --- a/qai_hub_models/models/densenet121/perf.yaml +++ b/qai_hub_models/models/densenet121/perf.yaml @@ -87,7 +87,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:50.487351Z' + timestamp: '2024-05-29T18:59:40.378424Z' - torchscript_onnx_tflite: inference_time: 1318.0 throughput: 758.7253414264036 @@ -140,7 +140,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:50.487479Z' + timestamp: '2024-05-29T18:59:40.378568Z' - torchscript_onnx_tflite: inference_time: 1941.0 throughput: 515.1983513652756 @@ -178,7 +178,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:50.487569Z' + timestamp: '2024-05-29T18:59:40.378662Z' - torchscript_onnx_qnn: inference_time: 2221.0 throughput: 450.24763619990995 @@ -209,21 +209,6 @@ models: total_layers: 374 job_id: jqpyd8d4p job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 15829.0 - throughput: 63.1751847874155 - estimated_peak_memory_range: - min: 70479872 - max: 70479872 - primary_compute_unit: GPU - precision: fp16 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 1 - layers_on_cpu: 0 - total_layers: 1 - job_id: j2p0rorep - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -231,4 +216,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:50.487661Z' + timestamp: '2024-05-29T18:59:40.378764Z' diff --git a/qai_hub_models/models/detr_resnet101/export.py b/qai_hub_models/models/detr_resnet101/export.py index c04ab155..863e0358 100644 --- a/qai_hub_models/models/detr_resnet101/export.py +++ b/qai_hub_models/models/detr_resnet101/export.py @@ -183,11 +183,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/detr_resnet101/perf.yaml b/qai_hub_models/models/detr_resnet101/perf.yaml index 55d574d6..3bfa8d81 100644 --- a/qai_hub_models/models/detr_resnet101/perf.yaml +++ b/qai_hub_models/models/detr_resnet101/perf.yaml @@ -50,21 +50,6 @@ models: total_layers: 839 job_id: j1p87j785 job_status: Passed - torchscript_onnx_qnn: - inference_time: 21128.0 - throughput: 47.330556607345706 - estimated_peak_memory_range: - min: 2785280 - max: 34215384 - primary_compute_unit: NPU - precision: fp16 - layer_info: - layers_on_npu: 1084 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 1084 - job_id: j1glkwklp - job_status: Passed torchscript_onnx_ort: inference_time: 22705.0 throughput: 44.04316229905307 @@ -87,7 +72,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:50.518538Z' + timestamp: '2024-05-29T18:59:40.413162Z' - torchscript_onnx_tflite: inference_time: 17296.0 throughput: 57.816836262719704 @@ -103,21 +88,6 @@ models: total_layers: 839 job_id: jogky6yop job_status: Passed - torchscript_onnx_qnn: - inference_time: 15082.0 - throughput: 66.30420368651373 - estimated_peak_memory_range: - min: 2781184 - max: 330046848 - primary_compute_unit: NPU - precision: fp16 - layer_info: - layers_on_npu: 1084 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 1084 - job_id: jw561o17p - job_status: Passed torchscript_onnx_ort: inference_time: 16129.0 throughput: 62.000124000248 @@ -140,7 +110,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:50.518809Z' + timestamp: '2024-05-29T18:59:40.413365Z' - torchscript_onnx_tflite: inference_time: 24577.0 throughput: 40.68844854945681 @@ -156,21 +126,6 @@ models: total_layers: 839 job_id: jn5q242m5 job_status: Passed - torchscript_onnx_qnn: - inference_time: 21161.0 - throughput: 47.25674590047729 - estimated_peak_memory_range: - min: 36864 - max: 30852728 - primary_compute_unit: NPU - precision: fp16 - layer_info: - layers_on_npu: 1084 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 1084 - job_id: jwgovdvd5 - job_status: Passed reference_device_info: name: QCS8550 (Proxy) os: '12' @@ -178,23 +133,8 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:50.518990Z' - - torchscript_onnx_qnn: - inference_time: 31269.0 - throughput: 31.980555822060186 - estimated_peak_memory_range: - min: 2768896 - max: 2768896 - primary_compute_unit: NPU - precision: fp16 - layer_info: - layers_on_npu: 1084 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 1084 - job_id: j1p3momzg - job_status: Passed - torchscript_onnx_ort: + timestamp: '2024-05-29T18:59:40.413467Z' + - torchscript_onnx_ort: inference_time: 22988.0 throughput: 43.50095702105446 estimated_peak_memory_range: @@ -209,21 +149,6 @@ models: total_layers: 856 job_id: jlpev6v05 job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 194914.0 - throughput: 5.130467796053644 - estimated_peak_memory_range: - min: 280969216 - max: 280969216 - primary_compute_unit: CPU - precision: fp32 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 628 - total_layers: 628 - job_id: jygz7z76p - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -231,4 +156,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:50.519234Z' + timestamp: '2024-05-29T18:59:40.413572Z' diff --git a/qai_hub_models/models/detr_resnet101_dc5/export.py b/qai_hub_models/models/detr_resnet101_dc5/export.py index 1af165db..cefe54f7 100644 --- a/qai_hub_models/models/detr_resnet101_dc5/export.py +++ b/qai_hub_models/models/detr_resnet101_dc5/export.py @@ -183,11 +183,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/detr_resnet101_dc5/perf.yaml b/qai_hub_models/models/detr_resnet101_dc5/perf.yaml index 5ee9061b..06ea9ea9 100644 --- a/qai_hub_models/models/detr_resnet101_dc5/perf.yaml +++ b/qai_hub_models/models/detr_resnet101_dc5/perf.yaml @@ -50,21 +50,6 @@ models: total_layers: 840 job_id: jvgdv6vlg job_status: Passed - torchscript_onnx_qnn: - inference_time: 132534.0 - throughput: 7.5452336758869425 - estimated_peak_memory_range: - min: 61440 - max: 61351040 - primary_compute_unit: NPU - precision: fp16 - layer_info: - layers_on_npu: 1084 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 1084 - job_id: j0px1079g - job_status: Passed torchscript_onnx_ort: inference_time: 126534.0 throughput: 7.903014209619549 @@ -87,7 +72,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:50.550081Z' + timestamp: '2024-05-29T18:59:40.439417Z' - torchscript_onnx_tflite: inference_time: 107197.0 throughput: 9.32861927106169 @@ -103,21 +88,6 @@ models: total_layers: 840 job_id: jz57dojr5 job_status: Passed - torchscript_onnx_qnn: - inference_time: 100177.0 - throughput: 9.982331273645647 - estimated_peak_memory_range: - min: 226050048 - max: 577743616 - primary_compute_unit: NPU - precision: fp16 - layer_info: - layers_on_npu: 1084 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 1084 - job_id: jo5mz9wqp - job_status: Passed torchscript_onnx_ort: inference_time: 95203.0 throughput: 10.503870676344233 @@ -140,7 +110,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:50.550356Z' + timestamp: '2024-05-29T18:59:40.439619Z' - torchscript_onnx_tflite: inference_time: 141441.0 throughput: 7.0700857601402705 @@ -156,21 +126,6 @@ models: total_layers: 840 job_id: jqp4wexlg job_status: Passed - torchscript_onnx_qnn: - inference_time: 141461.0 - throughput: 7.069086179229611 - estimated_peak_memory_range: - min: 2785280 - max: 59757592 - primary_compute_unit: NPU - precision: fp16 - layer_info: - layers_on_npu: 1084 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 1084 - job_id: jep2mo7m5 - job_status: Passed reference_device_info: name: QCS8550 (Proxy) os: '12' @@ -178,23 +133,8 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:50.550538Z' - - torchscript_onnx_qnn: - inference_time: 172476.0 - throughput: 5.797908114752198 - estimated_peak_memory_range: - min: 2772992 - max: 2772992 - primary_compute_unit: NPU - precision: fp16 - layer_info: - layers_on_npu: 1084 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 1084 - job_id: jopryx4eg - job_status: Passed - torchscript_onnx_ort: + timestamp: '2024-05-29T18:59:40.439721Z' + - torchscript_onnx_ort: inference_time: 125955.0 throughput: 7.939343416299472 estimated_peak_memory_range: @@ -209,21 +149,6 @@ models: total_layers: 856 job_id: j1p87j385 job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: jogky6lop - job_status: Failed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -231,4 +156,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:50.550749Z' + timestamp: '2024-05-29T18:59:40.439826Z' diff --git a/qai_hub_models/models/detr_resnet50/export.py b/qai_hub_models/models/detr_resnet50/export.py index a277e892..f7bd5816 100644 --- a/qai_hub_models/models/detr_resnet50/export.py +++ b/qai_hub_models/models/detr_resnet50/export.py @@ -183,11 +183,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/detr_resnet50/perf.yaml b/qai_hub_models/models/detr_resnet50/perf.yaml index 112c6123..974d50cd 100644 --- a/qai_hub_models/models/detr_resnet50/perf.yaml +++ b/qai_hub_models/models/detr_resnet50/perf.yaml @@ -50,21 +50,6 @@ models: total_layers: 771 job_id: jw561o37p job_status: Passed - torchscript_onnx_qnn: - inference_time: 19349.0 - throughput: 51.68225748100677 - estimated_peak_memory_range: - min: 2785280 - max: 23297304 - primary_compute_unit: NPU - precision: fp16 - layer_info: - layers_on_npu: 863 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 863 - job_id: j1pvw21mg - job_status: Passed torchscript_onnx_ort: inference_time: 16647.0 throughput: 60.070883642698384 @@ -87,7 +72,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:50.581613Z' + timestamp: '2024-05-29T18:59:40.465610Z' - torchscript_onnx_tflite: inference_time: 14432.0 throughput: 69.29046563192905 @@ -103,21 +88,6 @@ models: total_layers: 771 job_id: j1p3mo4zg job_status: Passed - torchscript_onnx_qnn: - inference_time: 13641.0 - throughput: 73.30840847445202 - estimated_peak_memory_range: - min: 2781184 - max: 246317600 - primary_compute_unit: NPU - precision: fp16 - layer_info: - layers_on_npu: 863 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 863 - job_id: j7gjl308p - job_status: Passed torchscript_onnx_ort: inference_time: 11774.0 throughput: 84.93290300662477 @@ -140,7 +110,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:50.581857Z' + timestamp: '2024-05-29T18:59:40.465801Z' - torchscript_onnx_tflite: inference_time: 20845.0 throughput: 47.97313504437515 @@ -156,21 +126,6 @@ models: total_layers: 771 job_id: jwgovd1d5 job_status: Passed - torchscript_onnx_qnn: - inference_time: 19468.0 - throughput: 51.3663447709061 - estimated_peak_memory_range: - min: 2818048 - max: 22271808 - primary_compute_unit: NPU - precision: fp16 - layer_info: - layers_on_npu: 863 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 863 - job_id: jygz7zx6p - job_status: Passed reference_device_info: name: QCS8550 (Proxy) os: '12' @@ -178,23 +133,8 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:50.582014Z' - - torchscript_onnx_qnn: - inference_time: 22398.0 - throughput: 44.6468434681668 - estimated_peak_memory_range: - min: 2768896 - max: 2768896 - primary_compute_unit: NPU - precision: fp16 - layer_info: - layers_on_npu: 863 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 863 - job_id: jlpev6r05 - job_status: Passed - torchscript_onnx_ort: + timestamp: '2024-05-29T18:59:40.465902Z' + - torchscript_onnx_ort: inference_time: 17028.0 throughput: 58.72680291284942 estimated_peak_memory_range: @@ -209,21 +149,6 @@ models: total_layers: 737 job_id: jnp18odlg job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: jvgdv6rlg - job_status: Failed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -231,4 +156,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:50.582175Z' + timestamp: '2024-05-29T18:59:40.465999Z' diff --git a/qai_hub_models/models/detr_resnet50_dc5/export.py b/qai_hub_models/models/detr_resnet50_dc5/export.py index 2a578429..adfa7e7f 100644 --- a/qai_hub_models/models/detr_resnet50_dc5/export.py +++ b/qai_hub_models/models/detr_resnet50_dc5/export.py @@ -183,11 +183,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/detr_resnet50_dc5/perf.yaml b/qai_hub_models/models/detr_resnet50_dc5/perf.yaml index bbb52225..ee2eb6c6 100644 --- a/qai_hub_models/models/detr_resnet50_dc5/perf.yaml +++ b/qai_hub_models/models/detr_resnet50_dc5/perf.yaml @@ -50,21 +50,6 @@ models: total_layers: 772 job_id: jz57dovr5 job_status: Passed - torchscript_onnx_qnn: - inference_time: 132937.0 - throughput: 7.5223602157412905 - estimated_peak_memory_range: - min: 49152 - max: 54649976 - primary_compute_unit: NPU - precision: fp16 - layer_info: - layers_on_npu: 863 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 863 - job_id: jo5mz9vqp - job_status: Passed torchscript_onnx_ort: inference_time: 119136.0 throughput: 8.393768466290625 @@ -87,7 +72,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:50.612935Z' + timestamp: '2024-05-29T18:59:40.491723Z' - torchscript_onnx_tflite: inference_time: 101510.0 throughput: 9.851246182642104 @@ -103,21 +88,6 @@ models: total_layers: 772 job_id: jqp4wejlg job_status: Passed - torchscript_onnx_qnn: - inference_time: 92358.0 - throughput: 10.827432382684771 - estimated_peak_memory_range: - min: 2875392 - max: 289938976 - primary_compute_unit: NPU - precision: fp16 - layer_info: - layers_on_npu: 863 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 863 - job_id: jegne1rmg - job_status: Passed torchscript_onnx_ort: inference_time: 91238.0 throughput: 10.960345470089218 @@ -140,7 +110,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:50.613175Z' + timestamp: '2024-05-29T18:59:40.491907Z' - torchscript_onnx_tflite: inference_time: 134198.0 throughput: 7.451675881905841 @@ -156,21 +126,6 @@ models: total_layers: 772 job_id: j0px10e9g job_status: Passed - torchscript_onnx_qnn: - inference_time: 132867.0 - throughput: 7.526323315797 - estimated_peak_memory_range: - min: 155648 - max: 54547288 - primary_compute_unit: NPU - precision: fp16 - layer_info: - layers_on_npu: 863 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 863 - job_id: jep2mo3m5 - job_status: Passed reference_device_info: name: QCS8550 (Proxy) os: '12' @@ -178,23 +133,8 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:50.613334Z' - - torchscript_onnx_qnn: - inference_time: 165645.0 - throughput: 6.037006852002777 - estimated_peak_memory_range: - min: 2772992 - max: 2772992 - primary_compute_unit: NPU - precision: fp16 - layer_info: - layers_on_npu: 863 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 863 - job_id: jopryx1eg - job_status: Passed - torchscript_onnx_ort: + timestamp: '2024-05-29T18:59:40.492030Z' + - torchscript_onnx_ort: inference_time: 118988.0 throughput: 8.404208827780952 estimated_peak_memory_range: @@ -209,21 +149,6 @@ models: total_layers: 737 job_id: j1p87jw85 job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: jogky6rop - job_status: Failed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -231,4 +156,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:50.613496Z' + timestamp: '2024-05-29T18:59:40.492122Z' diff --git a/qai_hub_models/models/efficientnet_b0/export.py b/qai_hub_models/models/efficientnet_b0/export.py index 3cf8c0d6..7b0d5f3d 100644 --- a/qai_hub_models/models/efficientnet_b0/export.py +++ b/qai_hub_models/models/efficientnet_b0/export.py @@ -185,11 +185,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/efficientnet_b0/perf.yaml b/qai_hub_models/models/efficientnet_b0/perf.yaml index d2c2aa94..65190420 100644 --- a/qai_hub_models/models/efficientnet_b0/perf.yaml +++ b/qai_hub_models/models/efficientnet_b0/perf.yaml @@ -87,7 +87,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:50.644090Z' + timestamp: '2024-05-29T18:59:40.517590Z' - torchscript_onnx_tflite: inference_time: 1139.0 throughput: 877.9631255487269 @@ -140,7 +140,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:50.644204Z' + timestamp: '2024-05-29T18:59:40.517700Z' - torchscript_onnx_tflite: inference_time: 1630.0 throughput: 613.4969325153374 @@ -178,7 +178,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:50.644279Z' + timestamp: '2024-05-29T18:59:40.517774Z' - torchscript_onnx_qnn: inference_time: 1830.0 throughput: 546.448087431694 @@ -209,21 +209,6 @@ models: total_layers: 245 job_id: jmg94o9v5 job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 10329.0 - throughput: 96.8147933004163 - estimated_peak_memory_range: - min: 36925440 - max: 36925440 - primary_compute_unit: GPU - precision: fp16 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 1 - layers_on_cpu: 0 - total_layers: 1 - job_id: jnp18oqlg - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -231,4 +216,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:50.644357Z' + timestamp: '2024-05-29T18:59:40.517848Z' diff --git a/qai_hub_models/models/esrgan/export.py b/qai_hub_models/models/esrgan/export.py index e6a815ba..8603628c 100644 --- a/qai_hub_models/models/esrgan/export.py +++ b/qai_hub_models/models/esrgan/export.py @@ -183,11 +183,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/esrgan/perf.yaml b/qai_hub_models/models/esrgan/perf.yaml index 4f159df1..f84124f1 100644 --- a/qai_hub_models/models/esrgan/perf.yaml +++ b/qai_hub_models/models/esrgan/perf.yaml @@ -87,7 +87,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:50.675231Z' + timestamp: '2024-05-29T18:59:40.552188Z' - torchscript_onnx_tflite: inference_time: 53811.0 throughput: 18.58356098195536 @@ -140,7 +140,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:50.675519Z' + timestamp: '2024-05-29T18:59:40.552520Z' - torchscript_onnx_tflite: inference_time: 75584.0 throughput: 13.230313293818798 @@ -178,7 +178,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:50.675710Z' + timestamp: '2024-05-29T18:59:40.552756Z' - torchscript_onnx_qnn: inference_time: 73244.0 throughput: 13.652995467205505 @@ -209,21 +209,6 @@ models: total_layers: 1028 job_id: jopryx99g job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 890390.0 - throughput: 1.1231033592021473 - estimated_peak_memory_range: - min: 139395072 - max: 139395072 - primary_compute_unit: CPU - precision: fp32 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 675 - total_layers: 675 - job_id: jep2moj45 - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -231,4 +216,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:50.675971Z' + timestamp: '2024-05-29T18:59:40.552990Z' diff --git a/qai_hub_models/models/facebook_denoiser/export.py b/qai_hub_models/models/facebook_denoiser/export.py index 0b285b72..b03be595 100644 --- a/qai_hub_models/models/facebook_denoiser/export.py +++ b/qai_hub_models/models/facebook_denoiser/export.py @@ -169,11 +169,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/facebook_denoiser/perf.yaml b/qai_hub_models/models/facebook_denoiser/perf.yaml index c2f3b886..3fbd9376 100644 --- a/qai_hub_models/models/facebook_denoiser/perf.yaml +++ b/qai_hub_models/models/facebook_denoiser/perf.yaml @@ -72,7 +72,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:50.706546Z' + timestamp: '2024-05-29T18:59:40.586858Z' - torchscript_onnx_tflite: inference_time: 777305.0 throughput: 1.2864962916744394 @@ -110,7 +110,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:50.706606Z' + timestamp: '2024-05-29T18:59:40.586927Z' - torchscript_onnx_tflite: inference_time: 756067.0 throughput: 1.3226341051785093 @@ -133,7 +133,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:50.706638Z' + timestamp: '2024-05-29T18:59:40.586964Z' - torchscript_onnx_ort: inference_time: 16116345.0 throughput: 0.06204880821302845 @@ -149,21 +149,6 @@ models: total_layers: 178 job_id: j1glkw78p job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 126831.0 - throughput: 7.88450773075983 - estimated_peak_memory_range: - min: 140013568 - max: 140013568 - primary_compute_unit: CPU - precision: fp32 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 107 - total_layers: 107 - job_id: jw561ov0p - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -171,4 +156,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:50.706688Z' + timestamp: '2024-05-29T18:59:40.586998Z' diff --git a/qai_hub_models/models/fastsam_s/export.py b/qai_hub_models/models/fastsam_s/export.py index af49ddb0..2b188980 100644 --- a/qai_hub_models/models/fastsam_s/export.py +++ b/qai_hub_models/models/fastsam_s/export.py @@ -187,11 +187,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/fastsam_s/perf.yaml b/qai_hub_models/models/fastsam_s/perf.yaml index c29ab73e..8fc1e10c 100644 --- a/qai_hub_models/models/fastsam_s/perf.yaml +++ b/qai_hub_models/models/fastsam_s/perf.yaml @@ -50,21 +50,6 @@ models: total_layers: 288 job_id: j1p3mo8lg job_status: Passed - torchscript_onnx_qnn: - inference_time: 8210.0 - throughput: 121.8026796589525 - estimated_peak_memory_range: - min: 4939776 - max: 20764192 - primary_compute_unit: NPU - precision: fp16 - layer_info: - layers_on_npu: 286 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 286 - job_id: j7gjl31xp - job_status: Passed torchscript_onnx_ort: inference_time: 10946.0 throughput: 91.35757354284671 @@ -87,7 +72,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:50.729410Z' + timestamp: '2024-05-29T18:59:40.613022Z' - torchscript_onnx_tflite: inference_time: 6423.0 throughput: 155.6904873112253 @@ -103,21 +88,6 @@ models: total_layers: 288 job_id: jwgovdmx5 job_status: Passed - torchscript_onnx_qnn: - inference_time: 6173.0 - throughput: 161.99578810950916 - estimated_peak_memory_range: - min: 4931584 - max: 91060880 - primary_compute_unit: NPU - precision: fp16 - layer_info: - layers_on_npu: 286 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 286 - job_id: jlpev6215 - job_status: Passed torchscript_onnx_ort: inference_time: 8057.0 throughput: 124.11567580985478 @@ -140,7 +110,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:50.729519Z' + timestamp: '2024-05-29T18:59:40.613103Z' - torchscript_onnx_tflite: inference_time: 8777.0 throughput: 113.93414606357526 @@ -156,21 +126,6 @@ models: total_layers: 288 job_id: j1pvw24jg job_status: Passed - torchscript_onnx_qnn: - inference_time: 8251.0 - throughput: 121.19743061447097 - estimated_peak_memory_range: - min: 4935680 - max: 18209840 - primary_compute_unit: NPU - precision: fp16 - layer_info: - layers_on_npu: 286 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 286 - job_id: jz5w9y86p - job_status: Passed reference_device_info: name: QCS8550 (Proxy) os: '12' @@ -178,23 +133,8 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:50.729609Z' - - torchscript_onnx_qnn: - inference_time: 9289.0 - throughput: 107.65421466250403 - estimated_peak_memory_range: - min: 4939776 - max: 4939776 - primary_compute_unit: NPU - precision: fp16 - layer_info: - layers_on_npu: 286 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 286 - job_id: jygz7zwkp - job_status: Passed - torchscript_onnx_ort: + timestamp: '2024-05-29T18:59:40.613145Z' + - torchscript_onnx_ort: inference_time: 10792.0 throughput: 92.66123054114159 estimated_peak_memory_range: @@ -209,21 +149,6 @@ models: total_layers: 289 job_id: jvgdv68eg job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 63171.0 - throughput: 15.830048598249197 - estimated_peak_memory_range: - min: 183259136 - max: 183259136 - primary_compute_unit: GPU - precision: fp16 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 1 - layers_on_cpu: 0 - total_layers: 1 - job_id: jz57dokl5 - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -231,4 +156,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:50.729693Z' + timestamp: '2024-05-29T18:59:40.613189Z' diff --git a/qai_hub_models/models/fastsam_x/export.py b/qai_hub_models/models/fastsam_x/export.py index f4bc7300..32977477 100644 --- a/qai_hub_models/models/fastsam_x/export.py +++ b/qai_hub_models/models/fastsam_x/export.py @@ -187,11 +187,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/fastsam_x/perf.yaml b/qai_hub_models/models/fastsam_x/perf.yaml index 398efcad..eb05cebb 100644 --- a/qai_hub_models/models/fastsam_x/perf.yaml +++ b/qai_hub_models/models/fastsam_x/perf.yaml @@ -50,21 +50,6 @@ models: total_layers: 420 job_id: jqp4wemvg job_status: Passed - torchscript_onnx_qnn: - inference_time: 46457.0 - throughput: 21.52528144305487 - estimated_peak_memory_range: - min: 4939776 - max: 21010728 - primary_compute_unit: NPU - precision: fp16 - layer_info: - layers_on_npu: 418 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 418 - job_id: jegne1org - job_status: Passed torchscript_onnx_ort: inference_time: 50303.0 throughput: 19.879530047909668 @@ -87,7 +72,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:50.760532Z' + timestamp: '2024-05-29T18:59:40.639222Z' - torchscript_onnx_tflite: inference_time: 36166.0 throughput: 27.650279267820604 @@ -103,21 +88,6 @@ models: total_layers: 420 job_id: j0px1031g job_status: Passed - torchscript_onnx_qnn: - inference_time: 35429.0 - throughput: 28.225465014536116 - estimated_peak_memory_range: - min: 0 - max: 124375088 - primary_compute_unit: NPU - precision: fp16 - layer_info: - layers_on_npu: 418 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 418 - job_id: jopryxo9g - job_status: Passed torchscript_onnx_ort: inference_time: 36822.0 throughput: 27.1576774754223 @@ -140,7 +110,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:50.760689Z' + timestamp: '2024-05-29T18:59:40.639336Z' - torchscript_onnx_tflite: inference_time: 52994.0 throughput: 18.870060761595653 @@ -156,21 +126,6 @@ models: total_layers: 420 job_id: jo5mz9owp job_status: Passed - torchscript_onnx_qnn: - inference_time: 46279.0 - throughput: 21.60807277598911 - estimated_peak_memory_range: - min: 4976640 - max: 21524200 - primary_compute_unit: NPU - precision: fp16 - layer_info: - layers_on_npu: 418 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 418 - job_id: jqpyd8q7p - job_status: Passed reference_device_info: name: QCS8550 (Proxy) os: '12' @@ -178,23 +133,8 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:50.760785Z' - - torchscript_onnx_qnn: - inference_time: 57682.0 - throughput: 17.336430775631914 - estimated_peak_memory_range: - min: 4939776 - max: 4939776 - primary_compute_unit: NPU - precision: fp16 - layer_info: - layers_on_npu: 418 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 418 - job_id: jep2mo445 - job_status: Passed - torchscript_onnx_ort: + timestamp: '2024-05-29T18:59:40.639421Z' + - torchscript_onnx_ort: inference_time: 49386.0 throughput: 20.248653464544606 estimated_peak_memory_range: @@ -209,21 +149,6 @@ models: total_layers: 421 job_id: jogky6o2p job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 568923.0 - throughput: 1.75770710623406 - estimated_peak_memory_range: - min: 45703168 - max: 45703168 - primary_compute_unit: CPU - precision: fp32 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 293 - total_layers: 293 - job_id: jn5q24z45 - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -231,4 +156,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:50.760939Z' + timestamp: '2024-05-29T18:59:40.639483Z' diff --git a/qai_hub_models/models/fcn_resnet50/export.py b/qai_hub_models/models/fcn_resnet50/export.py index 820b8b5e..4c3eb67f 100644 --- a/qai_hub_models/models/fcn_resnet50/export.py +++ b/qai_hub_models/models/fcn_resnet50/export.py @@ -184,11 +184,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/fcn_resnet50/perf.yaml b/qai_hub_models/models/fcn_resnet50/perf.yaml index c1b68175..d6b15006 100644 --- a/qai_hub_models/models/fcn_resnet50/perf.yaml +++ b/qai_hub_models/models/fcn_resnet50/perf.yaml @@ -87,7 +87,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:50.791629Z' + timestamp: '2024-05-29T18:59:40.665232Z' - torchscript_onnx_tflite: inference_time: 30738.0 throughput: 32.533021016331574 @@ -140,7 +140,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:50.791693Z' + timestamp: '2024-05-29T18:59:40.665299Z' - torchscript_onnx_tflite: inference_time: 42133.0 throughput: 23.734364987064772 @@ -178,7 +178,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:50.791735Z' + timestamp: '2024-05-29T18:59:40.665342Z' - torchscript_onnx_qnn: inference_time: 68527.0 throughput: 14.59278824404979 @@ -209,21 +209,6 @@ models: total_layers: 129 job_id: jmg94oxl5 job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: jnp18ov2g - job_status: Failed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -231,4 +216,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:50.791810Z' + timestamp: '2024-05-29T18:59:40.665389Z' diff --git a/qai_hub_models/models/fcn_resnet50_quantized/export.py b/qai_hub_models/models/fcn_resnet50_quantized/export.py index 19a22b4c..943f9fe5 100644 --- a/qai_hub_models/models/fcn_resnet50_quantized/export.py +++ b/qai_hub_models/models/fcn_resnet50_quantized/export.py @@ -192,11 +192,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/fcn_resnet50_quantized/perf.yaml b/qai_hub_models/models/fcn_resnet50_quantized/perf.yaml index 7ca993c6..acff4279 100644 --- a/qai_hub_models/models/fcn_resnet50_quantized/perf.yaml +++ b/qai_hub_models/models/fcn_resnet50_quantized/perf.yaml @@ -93,7 +93,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:50.822434Z' + timestamp: '2024-05-29T18:59:40.699656Z' - torchscript_onnx_tflite: inference_time: 10017.0 throughput: 99.83028850953379 @@ -146,7 +146,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:50.822489Z' + timestamp: '2024-05-29T18:59:40.699713Z' - torchscript_onnx_tflite: inference_time: 14106.0 throughput: 70.8918190840777 @@ -184,7 +184,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:50.822526Z' + timestamp: '2024-05-29T18:59:40.699752Z' - torchscript_onnx_tflite: inference_time: 89233.0 throughput: 11.20661638631448 @@ -222,7 +222,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs6490 - timestamp: '2024-05-24T06:12:50.822576Z' + timestamp: '2024-05-29T18:59:40.699790Z' - torchscript_onnx_tflite: inference_time: 752252.0 throughput: 1.3293417631325672 @@ -245,7 +245,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8250 - timestamp: '2024-05-24T06:12:50.822599Z' + timestamp: '2024-05-29T18:59:40.699812Z' - torchscript_onnx_qnn: inference_time: 16847.0 throughput: 59.357749154152074 @@ -276,21 +276,6 @@ models: total_layers: 82 job_id: jw561od0p job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 842652.0 - throughput: 1.1867295158618267 - estimated_peak_memory_range: - min: 69771264 - max: 69771264 - primary_compute_unit: CPU - precision: fp32 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 85 - total_layers: 85 - job_id: j1p3mowlg - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -298,4 +283,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:50.822665Z' + timestamp: '2024-05-29T18:59:40.699852Z' diff --git a/qai_hub_models/models/ffnet_122ns_lowres/export.py b/qai_hub_models/models/ffnet_122ns_lowres/export.py index 91065fda..53651a1f 100644 --- a/qai_hub_models/models/ffnet_122ns_lowres/export.py +++ b/qai_hub_models/models/ffnet_122ns_lowres/export.py @@ -183,11 +183,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/ffnet_122ns_lowres/perf.yaml b/qai_hub_models/models/ffnet_122ns_lowres/perf.yaml index 5bca2ff9..4c3cbe64 100644 --- a/qai_hub_models/models/ffnet_122ns_lowres/perf.yaml +++ b/qai_hub_models/models/ffnet_122ns_lowres/perf.yaml @@ -87,7 +87,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:50.862230Z' + timestamp: '2024-05-29T18:59:40.743297Z' - torchscript_onnx_tflite: inference_time: 7344.0 throughput: 136.16557734204792 @@ -140,7 +140,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:50.862356Z' + timestamp: '2024-05-29T18:59:40.743426Z' - torchscript_onnx_tflite: inference_time: 10684.0 throughput: 93.59790340696368 @@ -178,7 +178,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:50.862429Z' + timestamp: '2024-05-29T18:59:40.743508Z' - torchscript_onnx_qnn: inference_time: 17476.0 throughput: 57.221332112611584 @@ -209,21 +209,6 @@ models: total_layers: 350 job_id: j0px10x1g job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 25754.0 - throughput: 38.82891977945174 - estimated_peak_memory_range: - min: 85336064 - max: 85336064 - primary_compute_unit: GPU - precision: fp16 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 1 - layers_on_cpu: 0 - total_layers: 1 - job_id: jo5mz98wp - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -231,4 +216,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:50.862518Z' + timestamp: '2024-05-29T18:59:40.743602Z' diff --git a/qai_hub_models/models/ffnet_40s/export.py b/qai_hub_models/models/ffnet_40s/export.py index fb777ef7..31513e00 100644 --- a/qai_hub_models/models/ffnet_40s/export.py +++ b/qai_hub_models/models/ffnet_40s/export.py @@ -183,11 +183,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/ffnet_40s/perf.yaml b/qai_hub_models/models/ffnet_40s/perf.yaml index d15c1820..5e012497 100644 --- a/qai_hub_models/models/ffnet_40s/perf.yaml +++ b/qai_hub_models/models/ffnet_40s/perf.yaml @@ -87,7 +87,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:50.893315Z' + timestamp: '2024-05-29T18:59:40.778232Z' - torchscript_onnx_tflite: inference_time: 16624.0 throughput: 60.15399422521656 @@ -140,7 +140,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:50.893380Z' + timestamp: '2024-05-29T18:59:40.778307Z' - torchscript_onnx_tflite: inference_time: 22986.0 throughput: 43.50474201687984 @@ -178,7 +178,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:50.893429Z' + timestamp: '2024-05-29T18:59:40.778354Z' - torchscript_onnx_qnn: inference_time: 23238.0 throughput: 43.03296324984939 @@ -209,21 +209,6 @@ models: total_layers: 142 job_id: j1p3mo6lg job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 138142.0 - throughput: 7.238928059532944 - estimated_peak_memory_range: - min: 208830464 - max: 208830464 - primary_compute_unit: CPU - precision: fp32 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 67 - total_layers: 67 - job_id: jwgovd8x5 - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -231,4 +216,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:50.893501Z' + timestamp: '2024-05-29T18:59:40.778407Z' diff --git a/qai_hub_models/models/ffnet_40s_quantized/export.py b/qai_hub_models/models/ffnet_40s_quantized/export.py index bb295d3c..91d168bb 100644 --- a/qai_hub_models/models/ffnet_40s_quantized/export.py +++ b/qai_hub_models/models/ffnet_40s_quantized/export.py @@ -191,11 +191,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/ffnet_40s_quantized/perf.yaml b/qai_hub_models/models/ffnet_40s_quantized/perf.yaml index 0b49c9c8..4413dde3 100644 --- a/qai_hub_models/models/ffnet_40s_quantized/perf.yaml +++ b/qai_hub_models/models/ffnet_40s_quantized/perf.yaml @@ -56,21 +56,6 @@ models: total_layers: 97 job_id: j1pvw27jg job_status: Passed - torchscript_onnx_qnn: - inference_time: 4372.0 - throughput: 228.72827081427263 - estimated_peak_memory_range: - min: 8425472 - max: 23814016 - primary_compute_unit: NPU - precision: int8 - layer_info: - layers_on_npu: 89 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 89 - job_id: jmg94oml5 - job_status: Passed torchscript_onnx_ort: inference_time: 11412.0 throughput: 87.62705923589205 @@ -93,7 +78,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:50.924070Z' + timestamp: '2024-05-29T18:59:40.815172Z' - torchscript_onnx_tflite: inference_time: 4740.0 throughput: 210.9704641350211 @@ -109,21 +94,6 @@ models: total_layers: 97 job_id: j7gjl3qxp job_status: Passed - torchscript_onnx_qnn: - inference_time: 3167.0 - throughput: 315.75623618566465 - estimated_peak_memory_range: - min: 6348800 - max: 57501040 - primary_compute_unit: NPU - precision: int8 - layer_info: - layers_on_npu: 89 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 89 - job_id: jnp18oj2g - job_status: Passed torchscript_onnx_ort: inference_time: 8772.0 throughput: 113.99908800729594 @@ -146,7 +116,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:50.924131Z' + timestamp: '2024-05-29T18:59:40.815222Z' - torchscript_onnx_tflite: inference_time: 6448.0 throughput: 155.08684863523573 @@ -162,21 +132,6 @@ models: total_layers: 97 job_id: jlpev6y15 job_status: Passed - torchscript_onnx_qnn: - inference_time: 4328.0 - throughput: 231.0536044362292 - estimated_peak_memory_range: - min: 6320128 - max: 20448704 - primary_compute_unit: NPU - precision: int8 - layer_info: - layers_on_npu: 89 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 89 - job_id: jz5w9y73p - job_status: Passed reference_device_info: name: QCS8550 (Proxy) os: '12' @@ -184,7 +139,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:50.924173Z' + timestamp: '2024-05-29T18:59:40.815247Z' - torchscript_onnx_tflite: inference_time: 34910.0 throughput: 28.64508736751647 @@ -200,21 +155,6 @@ models: total_layers: 97 job_id: jygz7znkp job_status: Passed - torchscript_onnx_qnn: - inference_time: 26924.0 - throughput: 37.1415837171297 - estimated_peak_memory_range: - min: 6111232 - max: 53532960 - primary_compute_unit: NPU - precision: int8 - layer_info: - layers_on_npu: 89 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 89 - job_id: jmg94omw5 - job_status: Passed reference_device_info: name: RB3 Gen 2 (Proxy) os: '12' @@ -222,7 +162,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs6490 - timestamp: '2024-05-24T06:12:50.924235Z' + timestamp: '2024-05-29T18:59:40.815272Z' - torchscript_onnx_tflite: inference_time: 189525.0 throughput: 5.276348766653475 @@ -245,23 +185,8 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8250 - timestamp: '2024-05-24T06:12:50.924258Z' - - torchscript_onnx_qnn: - inference_time: 5223.0 - throughput: 191.46084625694044 - estimated_peak_memory_range: - min: 6303744 - max: 6303744 - primary_compute_unit: NPU - precision: int8 - layer_info: - layers_on_npu: 89 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 89 - job_id: jvgdv63eg - job_status: Passed - torchscript_onnx_ort: + timestamp: '2024-05-29T18:59:40.815296Z' + - torchscript_onnx_ort: inference_time: 10833.0 throughput: 92.31053263177328 estimated_peak_memory_range: @@ -276,21 +201,6 @@ models: total_layers: 94 job_id: jz57do4v5 job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 570899.0 - throughput: 1.7516233169089455 - estimated_peak_memory_range: - min: 204230656 - max: 204230656 - primary_compute_unit: GPU - precision: int8 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 1 - layers_on_cpu: 0 - total_layers: 1 - job_id: jqp4we18g - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -298,4 +208,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:50.924302Z' + timestamp: '2024-05-29T18:59:40.815322Z' diff --git a/qai_hub_models/models/ffnet_54s/export.py b/qai_hub_models/models/ffnet_54s/export.py index c81a8c18..78f844ff 100644 --- a/qai_hub_models/models/ffnet_54s/export.py +++ b/qai_hub_models/models/ffnet_54s/export.py @@ -183,11 +183,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/ffnet_54s/perf.yaml b/qai_hub_models/models/ffnet_54s/perf.yaml index 179311e4..2c8c14fb 100644 --- a/qai_hub_models/models/ffnet_54s/perf.yaml +++ b/qai_hub_models/models/ffnet_54s/perf.yaml @@ -87,7 +87,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:50.963638Z' + timestamp: '2024-05-29T18:59:40.847740Z' - torchscript_onnx_tflite: inference_time: 18458.0 throughput: 54.17705060136526 @@ -140,7 +140,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:50.963710Z' + timestamp: '2024-05-29T18:59:40.847824Z' - torchscript_onnx_tflite: inference_time: 25423.0 throughput: 39.33446092121307 @@ -178,7 +178,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:50.963761Z' + timestamp: '2024-05-29T18:59:40.847877Z' - torchscript_onnx_qnn: inference_time: 25826.0 throughput: 38.72066909316193 @@ -209,21 +209,6 @@ models: total_layers: 177 job_id: jn5q24yn5 job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 231904.0 - throughput: 4.3121291568925075 - estimated_peak_memory_range: - min: 341209088 - max: 341209088 - primary_compute_unit: GPU - precision: fp16 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 1 - layers_on_cpu: 0 - total_layers: 1 - job_id: j1glkwxjp - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -231,4 +216,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:50.963828Z' + timestamp: '2024-05-29T18:59:40.847941Z' diff --git a/qai_hub_models/models/ffnet_54s_quantized/export.py b/qai_hub_models/models/ffnet_54s_quantized/export.py index ba8ba6eb..ec7c47fc 100644 --- a/qai_hub_models/models/ffnet_54s_quantized/export.py +++ b/qai_hub_models/models/ffnet_54s_quantized/export.py @@ -191,11 +191,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/ffnet_54s_quantized/perf.yaml b/qai_hub_models/models/ffnet_54s_quantized/perf.yaml index 36a08286..1b654591 100644 --- a/qai_hub_models/models/ffnet_54s_quantized/perf.yaml +++ b/qai_hub_models/models/ffnet_54s_quantized/perf.yaml @@ -56,21 +56,6 @@ models: total_layers: 118 job_id: jw561o76p job_status: Passed - torchscript_onnx_qnn: - inference_time: 4948.0 - throughput: 202.1018593371059 - estimated_peak_memory_range: - min: 6340608 - max: 16584168 - primary_compute_unit: NPU - precision: int8 - layer_info: - layers_on_npu: 110 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 110 - job_id: jlpev67o5 - job_status: Passed torchscript_onnx_ort: inference_time: 11873.0 throughput: 84.22471153036301 @@ -93,7 +78,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:50.994861Z' + timestamp: '2024-05-29T18:59:40.882272Z' - torchscript_onnx_tflite: inference_time: 5175.0 throughput: 193.23671497584542 @@ -109,21 +94,6 @@ models: total_layers: 118 job_id: j1p3mo93g job_status: Passed - torchscript_onnx_qnn: - inference_time: 3622.0 - throughput: 276.09055770292656 - estimated_peak_memory_range: - min: 6307840 - max: 64549792 - primary_compute_unit: NPU - precision: int8 - layer_info: - layers_on_npu: 110 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 110 - job_id: jygz7zlop - job_status: Passed torchscript_onnx_ort: inference_time: 8976.0 throughput: 111.40819964349376 @@ -146,7 +116,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:50.994924Z' + timestamp: '2024-05-29T18:59:40.882322Z' - torchscript_onnx_tflite: inference_time: 7096.0 throughput: 140.92446448703495 @@ -162,21 +132,6 @@ models: total_layers: 118 job_id: jwgovdrq5 job_status: Passed - torchscript_onnx_qnn: - inference_time: 4982.0 - throughput: 200.7226013649137 - estimated_peak_memory_range: - min: 6152192 - max: 18613504 - primary_compute_unit: NPU - precision: int8 - layer_info: - layers_on_npu: 110 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 110 - job_id: jmg94ozw5 - job_status: Passed reference_device_info: name: QCS8550 (Proxy) os: '12' @@ -184,7 +139,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:50.994969Z' + timestamp: '2024-05-29T18:59:40.882348Z' - torchscript_onnx_tflite: inference_time: 39841.0 throughput: 25.09977159207851 @@ -200,21 +155,6 @@ models: total_layers: 118 job_id: j1pvw2lkg job_status: Passed - torchscript_onnx_qnn: - inference_time: 31044.0 - throughput: 32.21234377013271 - estimated_peak_memory_range: - min: 6332416 - max: 61553824 - primary_compute_unit: NPU - precision: int8 - layer_info: - layers_on_npu: 110 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 110 - job_id: jnp18on8g - job_status: Passed reference_device_info: name: RB3 Gen 2 (Proxy) os: '12' @@ -222,7 +162,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs6490 - timestamp: '2024-05-24T06:12:50.995019Z' + timestamp: '2024-05-29T18:59:40.882373Z' - torchscript_onnx_tflite: inference_time: 204729.0 throughput: 4.884505858964777 @@ -245,23 +185,8 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8250 - timestamp: '2024-05-24T06:12:50.995043Z' - - torchscript_onnx_qnn: - inference_time: 5964.0 - throughput: 167.6727028839705 - estimated_peak_memory_range: - min: 6303744 - max: 6303744 - primary_compute_unit: NPU - precision: int8 - layer_info: - layers_on_npu: 110 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 110 - job_id: jz5w9yl3p - job_status: Passed - torchscript_onnx_ort: + timestamp: '2024-05-29T18:59:40.882397Z' + - torchscript_onnx_ort: inference_time: 11333.0 throughput: 88.23788934968675 estimated_peak_memory_range: @@ -276,21 +201,6 @@ models: total_layers: 115 job_id: jqp4wey8g job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 656327.0 - throughput: 1.523630751134724 - estimated_peak_memory_range: - min: 204234752 - max: 204234752 - primary_compute_unit: GPU - precision: int8 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 1 - layers_on_cpu: 0 - total_layers: 1 - job_id: j0px10l3g - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -298,4 +208,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:50.995089Z' + timestamp: '2024-05-29T18:59:40.882423Z' diff --git a/qai_hub_models/models/ffnet_78s/export.py b/qai_hub_models/models/ffnet_78s/export.py index c75317a1..01600621 100644 --- a/qai_hub_models/models/ffnet_78s/export.py +++ b/qai_hub_models/models/ffnet_78s/export.py @@ -183,11 +183,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/ffnet_78s/perf.yaml b/qai_hub_models/models/ffnet_78s/perf.yaml index 898ec1f7..6c7da0ea 100644 --- a/qai_hub_models/models/ffnet_78s/perf.yaml +++ b/qai_hub_models/models/ffnet_78s/perf.yaml @@ -87,7 +87,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:51.034524Z' + timestamp: '2024-05-29T18:59:40.915116Z' - torchscript_onnx_tflite: inference_time: 21499.0 throughput: 46.513791339132055 @@ -140,7 +140,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:51.034617Z' + timestamp: '2024-05-29T18:59:40.915207Z' - torchscript_onnx_tflite: inference_time: 29503.0 throughput: 33.89485815001864 @@ -178,7 +178,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:51.034676Z' + timestamp: '2024-05-29T18:59:40.915264Z' - torchscript_onnx_qnn: inference_time: 32527.0 throughput: 30.743689857656715 @@ -209,21 +209,6 @@ models: total_layers: 237 job_id: j1glkwyjp job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 214768.0 - throughput: 4.65618714147359 - estimated_peak_memory_range: - min: 139489280 - max: 139489280 - primary_compute_unit: CPU - precision: fp32 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 105 - total_layers: 105 - job_id: jw561o86p - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -231,4 +216,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:51.034758Z' + timestamp: '2024-05-29T18:59:40.915328Z' diff --git a/qai_hub_models/models/ffnet_78s_lowres/export.py b/qai_hub_models/models/ffnet_78s_lowres/export.py index be2d2a1a..b0997e75 100644 --- a/qai_hub_models/models/ffnet_78s_lowres/export.py +++ b/qai_hub_models/models/ffnet_78s_lowres/export.py @@ -183,11 +183,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/ffnet_78s_lowres/perf.yaml b/qai_hub_models/models/ffnet_78s_lowres/perf.yaml index 77a2e4b7..aaa178ed 100644 --- a/qai_hub_models/models/ffnet_78s_lowres/perf.yaml +++ b/qai_hub_models/models/ffnet_78s_lowres/perf.yaml @@ -87,7 +87,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:51.065257Z' + timestamp: '2024-05-29T18:59:40.949492Z' - torchscript_onnx_tflite: inference_time: 7636.0 throughput: 130.95861707700368 @@ -140,23 +140,8 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:51.065348Z' - - torchscript_onnx_tflite: - inference_time: 10817.0 - throughput: 92.44707405010631 - estimated_peak_memory_range: - min: 692224 - max: 2481904 - primary_compute_unit: NPU - precision: fp16 - layer_info: - layers_on_npu: 149 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 149 - job_id: jygz7d7xp - job_status: Passed - torchscript_onnx_qnn: + timestamp: '2024-05-29T18:59:40.949582Z' + - torchscript_onnx_qnn: inference_time: 11366.0 throughput: 87.98169980644026 estimated_peak_memory_range: @@ -178,7 +163,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:51.065390Z' + timestamp: '2024-05-29T18:59:40.949622Z' - torchscript_onnx_qnn: inference_time: 20343.0 throughput: 49.1569581674286 @@ -209,21 +194,6 @@ models: total_layers: 238 job_id: jvgdv66rg job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 32679.0 - throughput: 30.60069157562961 - estimated_peak_memory_range: - min: 85352448 - max: 85352448 - primary_compute_unit: GPU - precision: fp16 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 1 - layers_on_cpu: 0 - total_layers: 1 - job_id: jz57doov5 - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -231,4 +201,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:51.065462Z' + timestamp: '2024-05-29T18:59:40.949692Z' diff --git a/qai_hub_models/models/ffnet_78s_quantized/export.py b/qai_hub_models/models/ffnet_78s_quantized/export.py index 404d4f3a..9b8ba13a 100644 --- a/qai_hub_models/models/ffnet_78s_quantized/export.py +++ b/qai_hub_models/models/ffnet_78s_quantized/export.py @@ -191,11 +191,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/ffnet_78s_quantized/perf.yaml b/qai_hub_models/models/ffnet_78s_quantized/perf.yaml index 4dceab12..b26e69f8 100644 --- a/qai_hub_models/models/ffnet_78s_quantized/perf.yaml +++ b/qai_hub_models/models/ffnet_78s_quantized/perf.yaml @@ -56,21 +56,6 @@ models: total_layers: 154 job_id: jqp4wee8g job_status: Passed - torchscript_onnx_qnn: - inference_time: 5979.0 - throughput: 167.25204883759827 - estimated_peak_memory_range: - min: 6344704 - max: 19127232 - primary_compute_unit: NPU - precision: int8 - layer_info: - layers_on_npu: 146 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 146 - job_id: jep2moor5 - job_status: Passed torchscript_onnx_ort: inference_time: 12055.0 throughput: 82.9531314807134 @@ -93,7 +78,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:51.095945Z' + timestamp: '2024-05-29T18:59:40.983404Z' - torchscript_onnx_tflite: inference_time: 6017.0 throughput: 166.19577862722286 @@ -109,21 +94,6 @@ models: total_layers: 154 job_id: j0px1003g job_status: Passed - torchscript_onnx_qnn: - inference_time: 4325.0 - throughput: 231.21387283236993 - estimated_peak_memory_range: - min: 74272768 - max: 145724208 - primary_compute_unit: NPU - precision: int8 - layer_info: - layers_on_npu: 146 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 146 - job_id: jqpyd888p - job_status: Passed torchscript_onnx_ort: inference_time: 9813.0 throughput: 101.9056353816366 @@ -146,7 +116,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:51.096017Z' + timestamp: '2024-05-29T18:59:40.983462Z' - torchscript_onnx_tflite: inference_time: 8357.0 throughput: 119.66016513102788 @@ -162,21 +132,6 @@ models: total_layers: 154 job_id: jo5mz99dp job_status: Passed - torchscript_onnx_qnn: - inference_time: 5950.0 - throughput: 168.0672268907563 - estimated_peak_memory_range: - min: 6324224 - max: 16961328 - primary_compute_unit: NPU - precision: int8 - layer_info: - layers_on_npu: 146 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 146 - job_id: j1p87jjk5 - job_status: Passed reference_device_info: name: QCS8550 (Proxy) os: '12' @@ -184,7 +139,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:51.096084Z' + timestamp: '2024-05-29T18:59:40.983492Z' - torchscript_onnx_tflite: inference_time: 44085.0 throughput: 22.683452421458547 @@ -200,21 +155,6 @@ models: total_layers: 154 job_id: jegne11kg job_status: Passed - torchscript_onnx_qnn: - inference_time: 36257.0 - throughput: 27.58088093333701 - estimated_peak_memory_range: - min: 6230016 - max: 71894352 - primary_compute_unit: NPU - precision: int8 - layer_info: - layers_on_npu: 146 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 146 - job_id: jogky66wp - job_status: Passed reference_device_info: name: RB3 Gen 2 (Proxy) os: '12' @@ -222,7 +162,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs6490 - timestamp: '2024-05-24T06:12:51.096130Z' + timestamp: '2024-05-29T18:59:40.983523Z' - torchscript_onnx_tflite: inference_time: 216166.0 throughput: 4.626074405780742 @@ -245,23 +185,8 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8250 - timestamp: '2024-05-24T06:12:51.096155Z' - - torchscript_onnx_qnn: - inference_time: 7087.0 - throughput: 141.10342881332016 - estimated_peak_memory_range: - min: 6303744 - max: 6303744 - primary_compute_unit: NPU - precision: int8 - layer_info: - layers_on_npu: 146 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 146 - job_id: j2p0roo9p - job_status: Passed - torchscript_onnx_ort: + timestamp: '2024-05-29T18:59:40.983553Z' + - torchscript_onnx_ort: inference_time: 11523.0 throughput: 86.78295582747549 estimated_peak_memory_range: @@ -276,21 +201,6 @@ models: total_layers: 151 job_id: jw561oo6p job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 1435080.0 - throughput: 0.6968252640967751 - estimated_peak_memory_range: - min: 241111040 - max: 241111040 - primary_compute_unit: CPU - precision: fp32 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 149 - total_layers: 149 - job_id: j1p3moo3g - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -298,4 +208,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:51.096227Z' + timestamp: '2024-05-29T18:59:40.983583Z' diff --git a/qai_hub_models/models/googlenet/export.py b/qai_hub_models/models/googlenet/export.py index 9496c3e3..a369bd62 100644 --- a/qai_hub_models/models/googlenet/export.py +++ b/qai_hub_models/models/googlenet/export.py @@ -185,11 +185,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/googlenet/perf.yaml b/qai_hub_models/models/googlenet/perf.yaml index 1fc6bbc8..20bc3165 100644 --- a/qai_hub_models/models/googlenet/perf.yaml +++ b/qai_hub_models/models/googlenet/perf.yaml @@ -87,7 +87,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:51.177119Z' + timestamp: '2024-05-29T18:59:41.061712Z' - torchscript_onnx_tflite: inference_time: 674.0 throughput: 1483.679525222552 @@ -140,7 +140,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:51.177189Z' + timestamp: '2024-05-29T18:59:41.061783Z' - torchscript_onnx_tflite: inference_time: 1047.0 throughput: 955.1098376313277 @@ -178,7 +178,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:51.177245Z' + timestamp: '2024-05-29T18:59:41.061828Z' - torchscript_onnx_qnn: inference_time: 1276.0 throughput: 783.6990595611285 @@ -209,21 +209,6 @@ models: total_layers: 145 job_id: j1p3mo73g job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 5891.0 - throughput: 169.75046681378373 - estimated_peak_memory_range: - min: 36888576 - max: 36888576 - primary_compute_unit: GPU - precision: fp16 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 1 - layers_on_cpu: 0 - total_layers: 1 - job_id: jwgovdwq5 - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -231,4 +216,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:51.177300Z' + timestamp: '2024-05-29T18:59:41.061878Z' diff --git a/qai_hub_models/models/googlenet_quantized/export.py b/qai_hub_models/models/googlenet_quantized/export.py index 395b8608..616b2243 100644 --- a/qai_hub_models/models/googlenet_quantized/export.py +++ b/qai_hub_models/models/googlenet_quantized/export.py @@ -193,11 +193,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/googlenet_quantized/perf.yaml b/qai_hub_models/models/googlenet_quantized/perf.yaml index ab774d7e..9d5cd175 100644 --- a/qai_hub_models/models/googlenet_quantized/perf.yaml +++ b/qai_hub_models/models/googlenet_quantized/perf.yaml @@ -93,7 +93,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:51.243304Z' + timestamp: '2024-05-29T18:59:41.096189Z' - torchscript_onnx_tflite: inference_time: 215.0 throughput: 4651.162790697675 @@ -146,7 +146,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:51.243364Z' + timestamp: '2024-05-29T18:59:41.096253Z' - torchscript_onnx_tflite: inference_time: 298.0 throughput: 3355.7046979865772 @@ -184,7 +184,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:51.243403Z' + timestamp: '2024-05-29T18:59:41.096292Z' - torchscript_onnx_tflite: inference_time: 977.0 throughput: 1023.5414534288639 @@ -222,7 +222,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs6490 - timestamp: '2024-05-24T06:12:51.243450Z' + timestamp: '2024-05-29T18:59:41.096330Z' - torchscript_onnx_tflite: inference_time: 5627.0 throughput: 177.7145903678692 @@ -245,7 +245,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8250 - timestamp: '2024-05-24T06:12:51.243473Z' + timestamp: '2024-05-29T18:59:41.096352Z' - torchscript_onnx_qnn: inference_time: 461.0 throughput: 2169.1973969631235 @@ -276,21 +276,6 @@ models: total_layers: 94 job_id: jz57do895 job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 2091.0 - throughput: 478.24007651841225 - estimated_peak_memory_range: - min: 17743872 - max: 17743872 - primary_compute_unit: CPU - precision: fp32 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 94 - total_layers: 94 - job_id: jqp4we21g - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -298,4 +283,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:51.243525Z' + timestamp: '2024-05-29T18:59:41.096393Z' diff --git a/qai_hub_models/models/hrnet_pose/export.py b/qai_hub_models/models/hrnet_pose/export.py index 80172428..c4f04ec8 100644 --- a/qai_hub_models/models/hrnet_pose/export.py +++ b/qai_hub_models/models/hrnet_pose/export.py @@ -184,11 +184,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/hrnet_pose/perf.yaml b/qai_hub_models/models/hrnet_pose/perf.yaml index 247a5042..b68df421 100644 --- a/qai_hub_models/models/hrnet_pose/perf.yaml +++ b/qai_hub_models/models/hrnet_pose/perf.yaml @@ -87,7 +87,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:51.282987Z' + timestamp: '2024-05-29T18:59:41.139934Z' - torchscript_onnx_tflite: inference_time: 2078.0 throughput: 481.23195380173246 @@ -140,7 +140,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:51.283199Z' + timestamp: '2024-05-29T18:59:41.140178Z' - torchscript_onnx_tflite: inference_time: 2876.0 throughput: 347.70514603616135 @@ -178,7 +178,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:51.283328Z' + timestamp: '2024-05-29T18:59:41.140315Z' - torchscript_onnx_qnn: inference_time: 3152.0 throughput: 317.25888324873097 @@ -209,21 +209,6 @@ models: total_layers: 749 job_id: jn5q24do5 job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 19418.0 - throughput: 51.498609537542485 - estimated_peak_memory_range: - min: 37539840 - max: 37539840 - primary_compute_unit: GPU - precision: fp16 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 1 - layers_on_cpu: 0 - total_layers: 1 - job_id: j1glkw9mp - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -231,4 +216,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:51.283486Z' + timestamp: '2024-05-29T18:59:41.140494Z' diff --git a/qai_hub_models/models/huggingface_wavlm_base_plus/export.py b/qai_hub_models/models/huggingface_wavlm_base_plus/export.py index 21b56dd4..5a4c60da 100644 --- a/qai_hub_models/models/huggingface_wavlm_base_plus/export.py +++ b/qai_hub_models/models/huggingface_wavlm_base_plus/export.py @@ -168,11 +168,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/huggingface_wavlm_base_plus/perf.yaml b/qai_hub_models/models/huggingface_wavlm_base_plus/perf.yaml index 71f5ecdd..48153308 100644 --- a/qai_hub_models/models/huggingface_wavlm_base_plus/perf.yaml +++ b/qai_hub_models/models/huggingface_wavlm_base_plus/perf.yaml @@ -50,36 +50,6 @@ models: total_layers: 811 job_id: jegne1dqg job_status: Passed - torchscript_onnx_qnn: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: jqpyd8klp - job_status: Failed - torchscript_onnx_ort: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: jogky6wnp - job_status: Failed reference_device_info: name: Samsung Galaxy S23 os: '13' @@ -87,7 +57,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:51.353892Z' + timestamp: '2024-05-29T18:59:41.210607Z' - torchscript_onnx_tflite: inference_time: 804134.0 throughput: 1.243573832221993 @@ -103,36 +73,6 @@ models: total_layers: 811 job_id: jopryxm7g job_status: Passed - torchscript_onnx_qnn: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: j2p0ro8np - job_status: Failed - torchscript_onnx_ort: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: jn5q24xo5 - job_status: Failed reference_device_info: name: Samsung Galaxy S24 os: '14' @@ -140,7 +80,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:51.354002Z' + timestamp: '2024-05-29T18:59:41.210708Z' - torchscript_onnx_tflite: inference_time: 932003.0 throughput: 1.0729579196633487 @@ -156,21 +96,6 @@ models: total_layers: 811 job_id: jep2moqq5 job_status: Passed - torchscript_onnx_qnn: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: j1p87jdo5 - job_status: Failed reference_device_info: name: QCS8550 (Proxy) os: '12' @@ -178,42 +103,12 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:51.354112Z' - - torchscript_onnx_ort: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: j1glkwdmp - job_status: Failed - torchscript_onnx_ort_dml_gpu: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: jw561oxyp - job_status: Failed - reference_device_info: + timestamp: '2024-05-29T18:59:41.210802Z' + - reference_device_info: name: Snapdragon X Elite CRD os: '11' form_factor: Compute os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:51.354133Z' + timestamp: '2024-05-29T18:59:41.210809Z' diff --git a/qai_hub_models/models/inception_v3/export.py b/qai_hub_models/models/inception_v3/export.py index 404d88f2..328357a4 100644 --- a/qai_hub_models/models/inception_v3/export.py +++ b/qai_hub_models/models/inception_v3/export.py @@ -185,11 +185,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/inception_v3/perf.yaml b/qai_hub_models/models/inception_v3/perf.yaml index 266853b2..93777204 100644 --- a/qai_hub_models/models/inception_v3/perf.yaml +++ b/qai_hub_models/models/inception_v3/perf.yaml @@ -87,7 +87,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:51.382754Z' + timestamp: '2024-05-29T18:59:41.228489Z' - torchscript_onnx_tflite: inference_time: 1028.0 throughput: 972.7626459143969 @@ -140,7 +140,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:51.382842Z' + timestamp: '2024-05-29T18:59:41.228583Z' - torchscript_onnx_tflite: inference_time: 1349.0 throughput: 741.2898443291327 @@ -178,7 +178,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:51.382902Z' + timestamp: '2024-05-29T18:59:41.228643Z' - torchscript_onnx_qnn: inference_time: 1519.0 throughput: 658.3278472679394 @@ -209,21 +209,6 @@ models: total_layers: 221 job_id: jvgdv6kzg job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 11363.0 - throughput: 88.00492827598346 - estimated_peak_memory_range: - min: 20066304 - max: 20066304 - primary_compute_unit: GPU - precision: fp16 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 1 - layers_on_cpu: 0 - total_layers: 1 - job_id: jz57dom95 - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -231,4 +216,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:51.382970Z' + timestamp: '2024-05-29T18:59:41.228713Z' diff --git a/qai_hub_models/models/inception_v3_quantized/export.py b/qai_hub_models/models/inception_v3_quantized/export.py index cacae613..cfd584a1 100644 --- a/qai_hub_models/models/inception_v3_quantized/export.py +++ b/qai_hub_models/models/inception_v3_quantized/export.py @@ -193,11 +193,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/inception_v3_quantized/perf.yaml b/qai_hub_models/models/inception_v3_quantized/perf.yaml index f1111da3..07c91205 100644 --- a/qai_hub_models/models/inception_v3_quantized/perf.yaml +++ b/qai_hub_models/models/inception_v3_quantized/perf.yaml @@ -93,7 +93,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:51.413420Z' + timestamp: '2024-05-29T18:59:41.263421Z' - torchscript_onnx_tflite: inference_time: 494.0 throughput: 2024.2914979757086 @@ -146,7 +146,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:51.413489Z' + timestamp: '2024-05-29T18:59:41.263500Z' - torchscript_onnx_tflite: inference_time: 613.0 throughput: 1631.3213703099511 @@ -184,7 +184,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:51.413537Z' + timestamp: '2024-05-29T18:59:41.263550Z' - torchscript_onnx_tflite: inference_time: 2343.0 throughput: 426.8032437046522 @@ -222,7 +222,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs6490 - timestamp: '2024-05-24T06:12:51.413591Z' + timestamp: '2024-05-29T18:59:41.263601Z' - torchscript_onnx_tflite: inference_time: 7547.0 throughput: 132.50298131707964 @@ -245,7 +245,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8250 - timestamp: '2024-05-24T06:12:51.413630Z' + timestamp: '2024-05-29T18:59:41.263629Z' - torchscript_onnx_qnn: inference_time: 710.0 throughput: 1408.4507042253522 @@ -276,21 +276,6 @@ models: total_layers: 137 job_id: jwgovdjk5 job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 21373.0 - throughput: 46.78800355588827 - estimated_peak_memory_range: - min: 20688896 - max: 20688896 - primary_compute_unit: GPU - precision: int8 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 1 - layers_on_cpu: 0 - total_layers: 1 - job_id: j1pvw2jrg - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -298,4 +283,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:51.413681Z' + timestamp: '2024-05-29T18:59:41.263679Z' diff --git a/qai_hub_models/models/lama_dilated/export.py b/qai_hub_models/models/lama_dilated/export.py index bb50420c..87478f48 100644 --- a/qai_hub_models/models/lama_dilated/export.py +++ b/qai_hub_models/models/lama_dilated/export.py @@ -187,11 +187,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/lama_dilated/perf.yaml b/qai_hub_models/models/lama_dilated/perf.yaml index d8b2f163..d59888e4 100644 --- a/qai_hub_models/models/lama_dilated/perf.yaml +++ b/qai_hub_models/models/lama_dilated/perf.yaml @@ -65,21 +65,6 @@ models: total_layers: 333 job_id: jvgdv6jzg job_status: Passed - torchscript_onnx_ort: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: jo5mz9j9p - job_status: Failed reference_device_info: name: Samsung Galaxy S23 os: '13' @@ -87,7 +72,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:51.453075Z' + timestamp: '2024-05-29T18:59:41.307510Z' - torchscript_onnx_tflite: inference_time: 59838.0 throughput: 16.7117884956048 @@ -118,21 +103,6 @@ models: total_layers: 333 job_id: jz57doq95 job_status: Passed - torchscript_onnx_ort: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: jegne1jqg - job_status: Failed reference_device_info: name: Samsung Galaxy S24 os: '14' @@ -140,7 +110,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:51.453178Z' + timestamp: '2024-05-29T18:59:41.307597Z' - torchscript_onnx_tflite: inference_time: 87348.0 throughput: 11.448459037413564 @@ -178,7 +148,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:51.453275Z' + timestamp: '2024-05-29T18:59:41.307681Z' - torchscript_onnx_qnn: inference_time: 92179.0 throughput: 10.848457891710693 @@ -194,36 +164,6 @@ models: total_layers: 333 job_id: jqp4wez1g job_status: Passed - torchscript_onnx_ort: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: jopryxz7g - job_status: Failed - torchscript_onnx_ort_dml_gpu: - inference_time: 862330.0 - throughput: 1.159648858325699 - estimated_peak_memory_range: - min: 279261184 - max: 279261184 - primary_compute_unit: CPU - precision: fp32 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 220 - total_layers: 220 - job_id: jep2mo2q5 - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -231,4 +171,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:51.453356Z' + timestamp: '2024-05-29T18:59:41.307729Z' diff --git a/qai_hub_models/models/litehrnet/export.py b/qai_hub_models/models/litehrnet/export.py index a13ca9e9..81e07f3a 100644 --- a/qai_hub_models/models/litehrnet/export.py +++ b/qai_hub_models/models/litehrnet/export.py @@ -169,11 +169,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/litehrnet/perf.yaml b/qai_hub_models/models/litehrnet/perf.yaml index 5139eb4f..87dcc012 100644 --- a/qai_hub_models/models/litehrnet/perf.yaml +++ b/qai_hub_models/models/litehrnet/perf.yaml @@ -50,21 +50,6 @@ models: total_layers: 1236 job_id: jqpyd89lp job_status: Passed - torchscript_onnx_ort: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: jn5q24jo5 - job_status: Failed reference_device_info: name: Samsung Galaxy S23 os: '13' @@ -72,7 +57,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:51.484113Z' + timestamp: '2024-05-29T18:59:41.334245Z' - torchscript_onnx_tflite: inference_time: 7918.0 throughput: 126.2945188178833 @@ -88,21 +73,6 @@ models: total_layers: 1236 job_id: j1p87jlo5 job_status: Passed - torchscript_onnx_ort: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: j1glkwnmp - job_status: Failed reference_device_info: name: Samsung Galaxy S24 os: '14' @@ -110,7 +80,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:51.484289Z' + timestamp: '2024-05-29T18:59:41.334395Z' - torchscript_onnx_tflite: inference_time: 11183.0 throughput: 89.42144326209424 @@ -133,42 +103,12 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:51.484405Z' - - torchscript_onnx_ort: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: jw561o6yp - job_status: Failed - torchscript_onnx_ort_dml_gpu: - inference_time: 36318.0 - throughput: 27.534555867613854 - estimated_peak_memory_range: - min: 9547776 - max: 9547776 - primary_compute_unit: CPU - precision: fp32 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 939 - total_layers: 939 - job_id: j1p3mokng - job_status: Passed - reference_device_info: + timestamp: '2024-05-29T18:59:41.334538Z' + - reference_device_info: name: Snapdragon X Elite CRD os: '11' form_factor: Compute os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:51.484504Z' + timestamp: '2024-05-29T18:59:41.334546Z' diff --git a/qai_hub_models/models/mediapipe_face/export.py b/qai_hub_models/models/mediapipe_face/export.py index efd2aee7..67ca40df 100644 --- a/qai_hub_models/models/mediapipe_face/export.py +++ b/qai_hub_models/models/mediapipe_face/export.py @@ -191,11 +191,11 @@ def export_model( # 5. Download the model assets to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) for component_name, compile_job in compile_jobs.items(): diff --git a/qai_hub_models/models/mediapipe_face/perf.yaml b/qai_hub_models/models/mediapipe_face/perf.yaml index 93f0531f..bf8dcabd 100644 --- a/qai_hub_models/models/mediapipe_face/perf.yaml +++ b/qai_hub_models/models/mediapipe_face/perf.yaml @@ -87,7 +87,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:51.506960Z' + timestamp: '2024-05-29T18:59:41.352382Z' - torchscript_onnx_tflite: inference_time: 575.0 throughput: 1739.1304347826087 @@ -140,7 +140,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:51.507029Z' + timestamp: '2024-05-29T18:59:41.352462Z' - torchscript_onnx_tflite: inference_time: 775.0 throughput: 1290.3225806451612 @@ -178,7 +178,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:51.507097Z' + timestamp: '2024-05-29T18:59:41.352513Z' - torchscript_onnx_qnn: inference_time: 931.0 throughput: 1074.1138560687432 @@ -209,21 +209,6 @@ models: total_layers: 147 job_id: jep2mo865 job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 5375.0 - throughput: 186.04651162790697 - estimated_peak_memory_range: - min: 12648448 - max: 12648448 - primary_compute_unit: GPU - precision: fp16 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 1 - layers_on_cpu: 0 - total_layers: 1 - job_id: j2p0roq0p - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -231,7 +216,7 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:51.507151Z' + timestamp: '2024-05-29T18:59:41.352570Z' - name: MediaPipeFaceLandmarkDetector performance_metrics: - torchscript_onnx_tflite: @@ -286,7 +271,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:51.507210Z' + timestamp: '2024-05-29T18:59:41.352636Z' - torchscript_onnx_tflite: inference_time: 247.0 throughput: 4048.582995951417 @@ -339,7 +324,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:51.507274Z' + timestamp: '2024-05-29T18:59:41.352701Z' - torchscript_onnx_tflite: inference_time: 311.0 throughput: 3215.434083601286 @@ -377,7 +362,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:51.507311Z' + timestamp: '2024-05-29T18:59:41.352746Z' - torchscript_onnx_qnn: inference_time: 520.0 throughput: 1923.076923076923 @@ -408,21 +393,6 @@ models: total_layers: 106 job_id: jqpyd8e0p job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 1185.0 - throughput: 843.8818565400844 - estimated_peak_memory_range: - min: 11816960 - max: 11816960 - primary_compute_unit: GPU - precision: fp16 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 1 - layers_on_cpu: 0 - total_layers: 1 - job_id: j1p87j9q5 - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -430,4 +400,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:51.507356Z' + timestamp: '2024-05-29T18:59:41.352791Z' diff --git a/qai_hub_models/models/mediapipe_hand/export.py b/qai_hub_models/models/mediapipe_hand/export.py index 9a457e01..df8302cd 100644 --- a/qai_hub_models/models/mediapipe_hand/export.py +++ b/qai_hub_models/models/mediapipe_hand/export.py @@ -191,11 +191,11 @@ def export_model( # 5. Download the model assets to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) for component_name, compile_job in compile_jobs.items(): diff --git a/qai_hub_models/models/mediapipe_hand/perf.yaml b/qai_hub_models/models/mediapipe_hand/perf.yaml index 59f8224d..53e99545 100644 --- a/qai_hub_models/models/mediapipe_hand/perf.yaml +++ b/qai_hub_models/models/mediapipe_hand/perf.yaml @@ -87,7 +87,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:51.565187Z' + timestamp: '2024-05-29T18:59:41.415401Z' - torchscript_onnx_tflite: inference_time: 703.0 throughput: 1422.475106685633 @@ -140,7 +140,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:51.565273Z' + timestamp: '2024-05-29T18:59:41.415495Z' - torchscript_onnx_tflite: inference_time: 954.0 throughput: 1048.2180293501049 @@ -178,7 +178,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:51.565328Z' + timestamp: '2024-05-29T18:59:41.415555Z' - torchscript_onnx_qnn: inference_time: 1058.0 throughput: 945.179584120983 @@ -209,21 +209,6 @@ models: total_layers: 196 job_id: j1p87jvq5 job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 16579.0 - throughput: 60.31726883406719 - estimated_peak_memory_range: - min: 18468864 - max: 18468864 - primary_compute_unit: CPU - precision: fp32 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 116 - total_layers: 116 - job_id: j1glkwr2p - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -231,7 +216,7 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:51.565403Z' + timestamp: '2024-05-29T18:59:41.415621Z' - name: MediaPipeHandLandmarkDetector performance_metrics: - torchscript_onnx_tflite: @@ -286,7 +271,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:51.565479Z' + timestamp: '2024-05-29T18:59:41.415714Z' - torchscript_onnx_tflite: inference_time: 896.0 throughput: 1116.0714285714287 @@ -339,7 +324,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:51.565554Z' + timestamp: '2024-05-29T18:59:41.415805Z' - torchscript_onnx_tflite: inference_time: 1200.0 throughput: 833.3333333333334 @@ -377,7 +362,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:51.565602Z' + timestamp: '2024-05-29T18:59:41.415867Z' - torchscript_onnx_qnn: inference_time: 1468.0 throughput: 681.1989100817439 @@ -408,21 +393,6 @@ models: total_layers: 209 job_id: jn5q24oe5 job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 18270.0 - throughput: 54.734537493158186 - estimated_peak_memory_range: - min: 872448 - max: 872448 - primary_compute_unit: CPU - precision: fp32 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 116 - total_layers: 116 - job_id: jw561olnp - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -430,4 +400,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:51.565673Z' + timestamp: '2024-05-29T18:59:41.415965Z' diff --git a/qai_hub_models/models/mediapipe_pose/export.py b/qai_hub_models/models/mediapipe_pose/export.py index 6625a35e..7bfbb700 100644 --- a/qai_hub_models/models/mediapipe_pose/export.py +++ b/qai_hub_models/models/mediapipe_pose/export.py @@ -191,11 +191,11 @@ def export_model( # 5. Download the model assets to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) for component_name, compile_job in compile_jobs.items(): diff --git a/qai_hub_models/models/mediapipe_pose/perf.yaml b/qai_hub_models/models/mediapipe_pose/perf.yaml index ac07f1d4..6bce9977 100644 --- a/qai_hub_models/models/mediapipe_pose/perf.yaml +++ b/qai_hub_models/models/mediapipe_pose/perf.yaml @@ -87,7 +87,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:51.623710Z' + timestamp: '2024-05-29T18:59:41.478689Z' - torchscript_onnx_tflite: inference_time: 595.0 throughput: 1680.672268907563 @@ -140,7 +140,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:51.623779Z' + timestamp: '2024-05-29T18:59:41.478764Z' - torchscript_onnx_tflite: inference_time: 851.0 throughput: 1175.0881316098707 @@ -178,7 +178,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:51.623822Z' + timestamp: '2024-05-29T18:59:41.478813Z' - torchscript_onnx_qnn: inference_time: 1047.0 throughput: 955.1098376313277 @@ -209,21 +209,6 @@ models: total_layers: 139 job_id: j1p87ryq5 job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 16089.0 - throughput: 62.15426689042203 - estimated_peak_memory_range: - min: 26910720 - max: 26910720 - primary_compute_unit: CPU - precision: fp32 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 81 - total_layers: 81 - job_id: jn5q21qe5 - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -231,7 +216,7 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:51.623885Z' + timestamp: '2024-05-29T18:59:41.478866Z' - name: MediaPipePoseLandmarkDetector performance_metrics: - torchscript_onnx_tflite: @@ -286,7 +271,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:51.623990Z' + timestamp: '2024-05-29T18:59:41.478982Z' - torchscript_onnx_tflite: inference_time: 885.0 throughput: 1129.9435028248588 @@ -339,7 +324,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:51.624087Z' + timestamp: '2024-05-29T18:59:41.479097Z' - torchscript_onnx_tflite: inference_time: 1229.0 throughput: 813.6696501220505 @@ -377,7 +362,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:51.624151Z' + timestamp: '2024-05-29T18:59:41.479173Z' - torchscript_onnx_qnn: inference_time: 1494.0 throughput: 669.3440428380187 @@ -408,21 +393,6 @@ models: total_layers: 304 job_id: jogky0xvp job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 6145.0 - throughput: 162.73393002441009 - estimated_peak_memory_range: - min: 20336640 - max: 20336640 - primary_compute_unit: GPU - precision: fp16 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 1 - layers_on_cpu: 0 - total_layers: 1 - job_id: j1glk8m2p - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -430,4 +400,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:51.624234Z' + timestamp: '2024-05-29T18:59:41.479256Z' diff --git a/qai_hub_models/models/mediapipe_selfie/export.py b/qai_hub_models/models/mediapipe_selfie/export.py index 220a4fb0..f137ad46 100644 --- a/qai_hub_models/models/mediapipe_selfie/export.py +++ b/qai_hub_models/models/mediapipe_selfie/export.py @@ -184,11 +184,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/mediapipe_selfie/perf.yaml b/qai_hub_models/models/mediapipe_selfie/perf.yaml index 51af7a48..913ec699 100644 --- a/qai_hub_models/models/mediapipe_selfie/perf.yaml +++ b/qai_hub_models/models/mediapipe_selfie/perf.yaml @@ -87,7 +87,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:51.682097Z' + timestamp: '2024-05-29T18:59:41.579849Z' - torchscript_onnx_tflite: inference_time: 523.0 throughput: 1912.0458891013384 @@ -140,7 +140,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:51.682175Z' + timestamp: '2024-05-29T18:59:41.579933Z' - torchscript_onnx_tflite: inference_time: 810.0 throughput: 1234.567901234568 @@ -178,7 +178,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:51.682238Z' + timestamp: '2024-05-29T18:59:41.580005Z' - torchscript_onnx_qnn: inference_time: 1023.0 throughput: 977.5171065493646 @@ -209,21 +209,6 @@ models: total_layers: 140 job_id: jvgdv416g job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 2862.0 - throughput: 349.4060097833683 - estimated_peak_memory_range: - min: 13230080 - max: 13230080 - primary_compute_unit: GPU - precision: fp16 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 1 - layers_on_cpu: 0 - total_layers: 1 - job_id: jqp4w4r2g - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -231,4 +216,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:51.682294Z' + timestamp: '2024-05-29T18:59:41.580058Z' diff --git a/qai_hub_models/models/midas/export.py b/qai_hub_models/models/midas/export.py index 8d650754..61e5504a 100644 --- a/qai_hub_models/models/midas/export.py +++ b/qai_hub_models/models/midas/export.py @@ -185,11 +185,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/midas/perf.yaml b/qai_hub_models/models/midas/perf.yaml index b471c922..1d16582c 100644 --- a/qai_hub_models/models/midas/perf.yaml +++ b/qai_hub_models/models/midas/perf.yaml @@ -87,7 +87,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:51.712791Z' + timestamp: '2024-05-29T18:59:41.614171Z' - torchscript_onnx_tflite: inference_time: 2439.0 throughput: 410.0041000410004 @@ -140,7 +140,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:51.712873Z' + timestamp: '2024-05-29T18:59:41.614261Z' - torchscript_onnx_tflite: inference_time: 3433.0 throughput: 291.29041654529567 @@ -178,7 +178,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:51.712936Z' + timestamp: '2024-05-29T18:59:41.614318Z' - torchscript_onnx_qnn: inference_time: 3591.0 throughput: 278.473962684489 @@ -209,21 +209,6 @@ models: total_layers: 199 job_id: jw561mynp job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 45335.0 - throughput: 22.058012573067167 - estimated_peak_memory_range: - min: 9842688 - max: 9842688 - primary_compute_unit: CPU - precision: fp32 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 112 - total_layers: 112 - job_id: j1p3m7jmg - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -231,4 +216,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:51.713007Z' + timestamp: '2024-05-29T18:59:41.614384Z' diff --git a/qai_hub_models/models/mnasnet05/export.py b/qai_hub_models/models/mnasnet05/export.py index 65a36009..389bfe46 100644 --- a/qai_hub_models/models/mnasnet05/export.py +++ b/qai_hub_models/models/mnasnet05/export.py @@ -185,11 +185,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/mnasnet05/perf.yaml b/qai_hub_models/models/mnasnet05/perf.yaml index 78e12a65..7f636ea6 100644 --- a/qai_hub_models/models/mnasnet05/perf.yaml +++ b/qai_hub_models/models/mnasnet05/perf.yaml @@ -87,7 +87,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:51.722925Z' + timestamp: '2024-05-29T18:59:41.648776Z' - torchscript_onnx_tflite: inference_time: 523.0 throughput: 1912.0458891013384 @@ -140,7 +140,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:51.722980Z' + timestamp: '2024-05-29T18:59:41.648842Z' - torchscript_onnx_tflite: inference_time: 780.0 throughput: 1282.051282051282 @@ -178,7 +178,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:51.723019Z' + timestamp: '2024-05-29T18:59:41.648882Z' - torchscript_onnx_qnn: inference_time: 941.0 throughput: 1062.6992561105208 @@ -209,21 +209,6 @@ models: total_layers: 104 job_id: jnp1814kg job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 2610.0 - throughput: 383.1417624521073 - estimated_peak_memory_range: - min: 11726848 - max: 11726848 - primary_compute_unit: GPU - precision: fp16 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 1 - layers_on_cpu: 0 - total_layers: 1 - job_id: jvgdv4xkg - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -231,4 +216,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:51.723082Z' + timestamp: '2024-05-29T18:59:41.648928Z' diff --git a/qai_hub_models/models/mobilenet_v2/export.py b/qai_hub_models/models/mobilenet_v2/export.py index 4dc8daad..4162e418 100644 --- a/qai_hub_models/models/mobilenet_v2/export.py +++ b/qai_hub_models/models/mobilenet_v2/export.py @@ -185,11 +185,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/mobilenet_v2/perf.yaml b/qai_hub_models/models/mobilenet_v2/perf.yaml index 04569096..dfb984fb 100644 --- a/qai_hub_models/models/mobilenet_v2/perf.yaml +++ b/qai_hub_models/models/mobilenet_v2/perf.yaml @@ -87,7 +87,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:51.753667Z' + timestamp: '2024-05-29T18:59:41.683566Z' - torchscript_onnx_tflite: inference_time: 642.0 throughput: 1557.632398753894 @@ -140,7 +140,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:51.753728Z' + timestamp: '2024-05-29T18:59:41.683632Z' - torchscript_onnx_tflite: inference_time: 939.0 throughput: 1064.9627263045793 @@ -178,7 +178,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:51.753770Z' + timestamp: '2024-05-29T18:59:41.683673Z' - torchscript_onnx_qnn: inference_time: 1557.0 throughput: 642.2607578676943 @@ -209,21 +209,6 @@ models: total_layers: 105 job_id: j1p87r2z5 job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 8115.0 - throughput: 123.22858903265558 - estimated_peak_memory_range: - min: 1404928 - max: 1404928 - primary_compute_unit: CPU - precision: fp32 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 57 - total_layers: 57 - job_id: jogky0vyp - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -231,4 +216,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:51.753852Z' + timestamp: '2024-05-29T18:59:41.683718Z' diff --git a/qai_hub_models/models/mobilenet_v2_quantized/export.py b/qai_hub_models/models/mobilenet_v2_quantized/export.py index 12d8723d..231631ce 100644 --- a/qai_hub_models/models/mobilenet_v2_quantized/export.py +++ b/qai_hub_models/models/mobilenet_v2_quantized/export.py @@ -193,11 +193,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/mobilenet_v2_quantized/perf.yaml b/qai_hub_models/models/mobilenet_v2_quantized/perf.yaml index 895df6af..499836d3 100644 --- a/qai_hub_models/models/mobilenet_v2_quantized/perf.yaml +++ b/qai_hub_models/models/mobilenet_v2_quantized/perf.yaml @@ -93,7 +93,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:51.784327Z' + timestamp: '2024-05-29T18:59:41.718178Z' - torchscript_onnx_tflite: inference_time: 207.0 throughput: 4830.917874396136 @@ -146,7 +146,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:51.784380Z' + timestamp: '2024-05-29T18:59:41.718236Z' - torchscript_onnx_tflite: inference_time: 317.0 throughput: 3154.5741324921137 @@ -184,7 +184,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:51.784413Z' + timestamp: '2024-05-29T18:59:41.718274Z' - torchscript_onnx_tflite: inference_time: 825.0 throughput: 1212.121212121212 @@ -222,7 +222,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs6490 - timestamp: '2024-05-24T06:12:51.784448Z' + timestamp: '2024-05-29T18:59:41.718310Z' - torchscript_onnx_tflite: inference_time: 7302.0 throughput: 136.9487811558477 @@ -245,7 +245,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8250 - timestamp: '2024-05-24T06:12:51.784493Z' + timestamp: '2024-05-29T18:59:41.718332Z' - torchscript_onnx_qnn: inference_time: 771.0 throughput: 1297.0168612191958 @@ -276,21 +276,6 @@ models: total_layers: 77 job_id: jvgdv4nkg job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 42826.0 - throughput: 23.350301218885722 - estimated_peak_memory_range: - min: 20054016 - max: 20054016 - primary_compute_unit: GPU - precision: int8 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 1 - layers_on_cpu: 0 - total_layers: 1 - job_id: jz57dn2q5 - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -298,4 +283,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:51.784533Z' + timestamp: '2024-05-29T18:59:41.718370Z' diff --git a/qai_hub_models/models/mobilenet_v3_large/export.py b/qai_hub_models/models/mobilenet_v3_large/export.py index 6ccd2370..40b17a01 100644 --- a/qai_hub_models/models/mobilenet_v3_large/export.py +++ b/qai_hub_models/models/mobilenet_v3_large/export.py @@ -185,11 +185,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/mobilenet_v3_large/perf.yaml b/qai_hub_models/models/mobilenet_v3_large/perf.yaml index 1a7b1a27..c0d437b8 100644 --- a/qai_hub_models/models/mobilenet_v3_large/perf.yaml +++ b/qai_hub_models/models/mobilenet_v3_large/perf.yaml @@ -87,7 +87,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:51.823745Z' + timestamp: '2024-05-29T18:59:41.761794Z' - torchscript_onnx_tflite: inference_time: 678.0 throughput: 1474.9262536873157 @@ -140,7 +140,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:51.823820Z' + timestamp: '2024-05-29T18:59:41.761876Z' - torchscript_onnx_tflite: inference_time: 1002.0 throughput: 998.003992015968 @@ -178,7 +178,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:51.823889Z' + timestamp: '2024-05-29T18:59:41.761928Z' - torchscript_onnx_qnn: inference_time: 1213.0 throughput: 824.4023083264633 @@ -209,21 +209,6 @@ models: total_layers: 162 job_id: jn5q21e75 job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 5539.0 - throughput: 180.53800324968407 - estimated_peak_memory_range: - min: 20070400 - max: 20070400 - primary_compute_unit: GPU - precision: fp16 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 1 - layers_on_cpu: 0 - total_layers: 1 - job_id: j1glk86ep - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -231,4 +216,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:51.823943Z' + timestamp: '2024-05-29T18:59:41.761984Z' diff --git a/qai_hub_models/models/mobilenet_v3_large_quantized/export.py b/qai_hub_models/models/mobilenet_v3_large_quantized/export.py index c115f693..ec90b231 100644 --- a/qai_hub_models/models/mobilenet_v3_large_quantized/export.py +++ b/qai_hub_models/models/mobilenet_v3_large_quantized/export.py @@ -193,11 +193,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/mobilenet_v3_large_quantized/perf.yaml b/qai_hub_models/models/mobilenet_v3_large_quantized/perf.yaml index bf6f38b2..a7a39b53 100644 --- a/qai_hub_models/models/mobilenet_v3_large_quantized/perf.yaml +++ b/qai_hub_models/models/mobilenet_v3_large_quantized/perf.yaml @@ -93,7 +93,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:51.854635Z' + timestamp: '2024-05-29T18:59:41.796254Z' - torchscript_onnx_tflite: inference_time: 255.0 throughput: 3921.5686274509803 @@ -146,7 +146,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:51.854708Z' + timestamp: '2024-05-29T18:59:41.796335Z' - torchscript_onnx_tflite: inference_time: 353.0 throughput: 2832.8611898016998 @@ -184,7 +184,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:51.854777Z' + timestamp: '2024-05-29T18:59:41.796386Z' - torchscript_onnx_tflite: inference_time: 1160.0 throughput: 862.0689655172414 @@ -222,7 +222,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs6490 - timestamp: '2024-05-24T06:12:51.854820Z' + timestamp: '2024-05-29T18:59:41.796435Z' - torchscript_onnx_tflite: inference_time: 6893.0 throughput: 145.0747134774409 @@ -245,7 +245,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8250 - timestamp: '2024-05-24T06:12:51.854844Z' + timestamp: '2024-05-29T18:59:41.796463Z' - torchscript_onnx_qnn: inference_time: 713.0 throughput: 1402.5245441795232 @@ -276,21 +276,6 @@ models: total_layers: 174 job_id: jo5mzkqyp job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 28734.0 - throughput: 34.80197675227953 - estimated_peak_memory_range: - min: 20717568 - max: 20717568 - primary_compute_unit: GPU - precision: int8 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 1 - layers_on_cpu: 0 - total_layers: 1 - job_id: jegneqmvg - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -298,4 +283,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:51.854900Z' + timestamp: '2024-05-29T18:59:41.796520Z' diff --git a/qai_hub_models/models/mobilenet_v3_small/export.py b/qai_hub_models/models/mobilenet_v3_small/export.py index b59dd0be..70b1d899 100644 --- a/qai_hub_models/models/mobilenet_v3_small/export.py +++ b/qai_hub_models/models/mobilenet_v3_small/export.py @@ -185,11 +185,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/mobilenet_v3_small/perf.yaml b/qai_hub_models/models/mobilenet_v3_small/perf.yaml index 854feca3..7bf1b957 100644 --- a/qai_hub_models/models/mobilenet_v3_small/perf.yaml +++ b/qai_hub_models/models/mobilenet_v3_small/perf.yaml @@ -87,7 +87,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:51.893860Z' + timestamp: '2024-05-29T18:59:41.840059Z' - torchscript_onnx_tflite: inference_time: 568.0 throughput: 1760.5633802816901 @@ -140,7 +140,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:51.893927Z' + timestamp: '2024-05-29T18:59:41.840135Z' - torchscript_onnx_tflite: inference_time: 836.0 throughput: 1196.1722488038276 @@ -178,7 +178,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:51.893977Z' + timestamp: '2024-05-29T18:59:41.840181Z' - torchscript_onnx_qnn: inference_time: 1065.0 throughput: 938.9671361502348 @@ -209,21 +209,6 @@ models: total_layers: 146 job_id: j1p3m71xg job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 3431.0 - throughput: 291.4602156805596 - estimated_peak_memory_range: - min: 11681792 - max: 11681792 - primary_compute_unit: GPU - precision: fp16 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 1 - layers_on_cpu: 0 - total_layers: 1 - job_id: jwgovwn45 - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -231,4 +216,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:51.894043Z' + timestamp: '2024-05-29T18:59:41.840231Z' diff --git a/qai_hub_models/models/openai_clip/export.py b/qai_hub_models/models/openai_clip/export.py index 18a3fee5..4cc801ca 100644 --- a/qai_hub_models/models/openai_clip/export.py +++ b/qai_hub_models/models/openai_clip/export.py @@ -191,11 +191,11 @@ def export_model( # 5. Download the model assets to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) for component_name, compile_job in compile_jobs.items(): diff --git a/qai_hub_models/models/openai_clip/perf.yaml b/qai_hub_models/models/openai_clip/perf.yaml index 8e0ae6db..da7d3a0d 100644 --- a/qai_hub_models/models/openai_clip/perf.yaml +++ b/qai_hub_models/models/openai_clip/perf.yaml @@ -87,7 +87,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:51.924796Z' + timestamp: '2024-05-29T18:59:41.874657Z' - torchscript_onnx_tflite: inference_time: 9373.0 throughput: 106.68942707777659 @@ -140,7 +140,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:51.924985Z' + timestamp: '2024-05-29T18:59:41.874827Z' - torchscript_onnx_tflite: inference_time: 13297.0 throughput: 75.2049334436339 @@ -178,7 +178,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:51.925093Z' + timestamp: '2024-05-29T18:59:41.874948Z' - torchscript_onnx_qnn: inference_time: 8459.0 throughput: 118.21728336682823 @@ -209,21 +209,6 @@ models: total_layers: 389 job_id: jogky03yp job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 9555.0 - throughput: 104.65724751439038 - estimated_peak_memory_range: - min: 561152 - max: 561152 - primary_compute_unit: CPU - precision: fp32 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 290 - total_layers: 290 - job_id: j1glk83ep - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -231,7 +216,7 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:51.925211Z' + timestamp: '2024-05-29T18:59:41.875049Z' - name: CLIPImageEncoder performance_metrics: - torchscript_onnx_tflite: @@ -286,7 +271,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:51.925350Z' + timestamp: '2024-05-29T18:59:41.875215Z' - torchscript_onnx_tflite: inference_time: 96011.0 throughput: 10.41547322702607 @@ -339,7 +324,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:51.925488Z' + timestamp: '2024-05-29T18:59:41.875378Z' - torchscript_onnx_tflite: inference_time: 126479.0 throughput: 7.906450873267499 @@ -377,7 +362,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:51.925587Z' + timestamp: '2024-05-29T18:59:41.875496Z' - torchscript_onnx_qnn: inference_time: 48879.0 throughput: 20.458683688291494 @@ -408,21 +393,6 @@ models: total_layers: 382 job_id: jn5q21375 job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: jw561mnvp - job_status: Failed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -430,4 +400,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:51.925678Z' + timestamp: '2024-05-29T18:59:41.875594Z' diff --git a/qai_hub_models/models/openpose/export.py b/qai_hub_models/models/openpose/export.py index 3324649a..c37d9c3d 100644 --- a/qai_hub_models/models/openpose/export.py +++ b/qai_hub_models/models/openpose/export.py @@ -185,11 +185,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/openpose/perf.yaml b/qai_hub_models/models/openpose/perf.yaml index f9196cbf..acbfb747 100644 --- a/qai_hub_models/models/openpose/perf.yaml +++ b/qai_hub_models/models/openpose/perf.yaml @@ -87,7 +87,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:51.983330Z' + timestamp: '2024-05-29T18:59:41.938307Z' - torchscript_onnx_tflite: inference_time: 8716.0 throughput: 114.73152822395595 @@ -140,7 +140,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:51.983415Z' + timestamp: '2024-05-29T18:59:41.938390Z' - torchscript_onnx_tflite: inference_time: 11834.0 throughput: 84.50228156160216 @@ -178,7 +178,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:51.983463Z' + timestamp: '2024-05-29T18:59:41.938443Z' - torchscript_onnx_qnn: inference_time: 14114.0 throughput: 70.85163667280715 @@ -209,21 +209,6 @@ models: total_layers: 189 job_id: jvgdv4okg job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 89367.0 - throughput: 11.18981279443195 - estimated_peak_memory_range: - min: 137478144 - max: 137478144 - primary_compute_unit: GPU - precision: fp16 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 1 - layers_on_cpu: 0 - total_layers: 1 - job_id: jz5w9zqjp - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -231,4 +216,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:51.983521Z' + timestamp: '2024-05-29T18:59:41.938505Z' diff --git a/qai_hub_models/models/posenet_mobilenet/export.py b/qai_hub_models/models/posenet_mobilenet/export.py index 61b98c15..9fa56461 100644 --- a/qai_hub_models/models/posenet_mobilenet/export.py +++ b/qai_hub_models/models/posenet_mobilenet/export.py @@ -183,11 +183,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/posenet_mobilenet/perf.yaml b/qai_hub_models/models/posenet_mobilenet/perf.yaml index d04b364a..e55112e4 100644 --- a/qai_hub_models/models/posenet_mobilenet/perf.yaml +++ b/qai_hub_models/models/posenet_mobilenet/perf.yaml @@ -87,7 +87,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:52.014199Z' + timestamp: '2024-05-29T18:59:41.972787Z' - torchscript_onnx_tflite: inference_time: 973.0 throughput: 1027.749229188078 @@ -140,7 +140,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:52.014248Z' + timestamp: '2024-05-29T18:59:41.972843Z' - torchscript_onnx_tflite: inference_time: 1391.0 throughput: 718.9072609633357 @@ -178,7 +178,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:52.014280Z' + timestamp: '2024-05-29T18:59:41.972876Z' - torchscript_onnx_qnn: inference_time: 1748.0 throughput: 572.0823798627002 @@ -209,21 +209,6 @@ models: total_layers: 70 job_id: jqpyd2d4p job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 19749.0 - throughput: 50.63547521393488 - estimated_peak_memory_range: - min: 52809728 - max: 52809728 - primary_compute_unit: CPU - precision: fp32 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 34 - total_layers: 34 - job_id: j2p0r9rep - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -231,4 +216,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:52.014349Z' + timestamp: '2024-05-29T18:59:41.972913Z' diff --git a/qai_hub_models/models/quicksrnetlarge/export.py b/qai_hub_models/models/quicksrnetlarge/export.py index 3b28fe33..d21f0eab 100644 --- a/qai_hub_models/models/quicksrnetlarge/export.py +++ b/qai_hub_models/models/quicksrnetlarge/export.py @@ -184,11 +184,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/quicksrnetlarge/perf.yaml b/qai_hub_models/models/quicksrnetlarge/perf.yaml index 8c045263..cb348dbc 100644 --- a/qai_hub_models/models/quicksrnetlarge/perf.yaml +++ b/qai_hub_models/models/quicksrnetlarge/perf.yaml @@ -87,7 +87,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:52.024298Z' + timestamp: '2024-05-29T18:59:42.007134Z' - torchscript_onnx_tflite: inference_time: 1797.0 throughput: 556.4830272676684 @@ -140,7 +140,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:52.024338Z' + timestamp: '2024-05-29T18:59:42.007179Z' - torchscript_onnx_tflite: inference_time: 2450.0 throughput: 408.16326530612247 @@ -178,7 +178,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:52.024364Z' + timestamp: '2024-05-29T18:59:42.007209Z' - torchscript_onnx_qnn: inference_time: 2946.0 throughput: 339.44331296673454 @@ -209,21 +209,6 @@ models: total_layers: 33 job_id: jygz7y76p job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 15003.0 - throughput: 66.65333599946678 - estimated_peak_memory_range: - min: 31277056 - max: 31277056 - primary_compute_unit: CPU - precision: fp32 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 15 - total_layers: 15 - job_id: jz5w9z9jp - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -231,4 +216,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:52.024398Z' + timestamp: '2024-05-29T18:59:42.007239Z' diff --git a/qai_hub_models/models/quicksrnetlarge_quantized/export.py b/qai_hub_models/models/quicksrnetlarge_quantized/export.py index c03cfabc..fcea80fa 100644 --- a/qai_hub_models/models/quicksrnetlarge_quantized/export.py +++ b/qai_hub_models/models/quicksrnetlarge_quantized/export.py @@ -192,11 +192,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/quicksrnetlarge_quantized/perf.yaml b/qai_hub_models/models/quicksrnetlarge_quantized/perf.yaml index 24d493fb..c42927e0 100644 --- a/qai_hub_models/models/quicksrnetlarge_quantized/perf.yaml +++ b/qai_hub_models/models/quicksrnetlarge_quantized/perf.yaml @@ -93,7 +93,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:52.054956Z' + timestamp: '2024-05-29T18:59:42.041532Z' - torchscript_onnx_tflite: inference_time: 1013.0 throughput: 987.1668311944719 @@ -146,7 +146,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:52.054996Z' + timestamp: '2024-05-29T18:59:42.041576Z' - torchscript_onnx_tflite: inference_time: 1409.0 throughput: 709.7232079488999 @@ -184,7 +184,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:52.055022Z' + timestamp: '2024-05-29T18:59:42.041604Z' - torchscript_onnx_tflite: inference_time: 3568.0 throughput: 280.2690582959641 @@ -222,7 +222,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs6490 - timestamp: '2024-05-24T06:12:52.055047Z' + timestamp: '2024-05-29T18:59:42.041632Z' - torchscript_onnx_tflite: inference_time: 34339.0 throughput: 29.121407146393313 @@ -245,7 +245,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8250 - timestamp: '2024-05-24T06:12:52.055065Z' + timestamp: '2024-05-29T18:59:42.041649Z' - torchscript_onnx_qnn: inference_time: 1090.0 throughput: 917.4311926605504 @@ -276,21 +276,6 @@ models: total_layers: 24 job_id: jn5q217m5 job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 210332.0 - throughput: 4.754388300401271 - estimated_peak_memory_range: - min: 32960512 - max: 32960512 - primary_compute_unit: GPU - precision: int8 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 1 - layers_on_cpu: 0 - total_layers: 1 - job_id: j1glk80lp - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -298,4 +283,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:52.055096Z' + timestamp: '2024-05-29T18:59:42.041677Z' diff --git a/qai_hub_models/models/quicksrnetmedium/export.py b/qai_hub_models/models/quicksrnetmedium/export.py index 9186328d..32a7b7ba 100644 --- a/qai_hub_models/models/quicksrnetmedium/export.py +++ b/qai_hub_models/models/quicksrnetmedium/export.py @@ -184,11 +184,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/quicksrnetmedium/perf.yaml b/qai_hub_models/models/quicksrnetmedium/perf.yaml index 93c22405..111b4ab6 100644 --- a/qai_hub_models/models/quicksrnetmedium/perf.yaml +++ b/qai_hub_models/models/quicksrnetmedium/perf.yaml @@ -87,7 +87,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:52.094437Z' + timestamp: '2024-05-29T18:59:42.084964Z' - torchscript_onnx_tflite: inference_time: 979.0 throughput: 1021.4504596527069 @@ -140,7 +140,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:52.094475Z' + timestamp: '2024-05-29T18:59:42.085007Z' - torchscript_onnx_tflite: inference_time: 1419.0 throughput: 704.7216349541931 @@ -178,7 +178,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:52.094499Z' + timestamp: '2024-05-29T18:59:42.085033Z' - torchscript_onnx_qnn: inference_time: 1228.0 throughput: 814.3322475570033 @@ -209,21 +209,6 @@ models: total_layers: 19 job_id: jnp181dlg job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 3229.0 - throughput: 309.6934035305048 - estimated_peak_memory_range: - min: 33107968 - max: 33107968 - primary_compute_unit: GPU - precision: fp16 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 1 - layers_on_cpu: 0 - total_layers: 1 - job_id: jvgdv4rlg - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -231,4 +216,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:52.094531Z' + timestamp: '2024-05-29T18:59:42.085060Z' diff --git a/qai_hub_models/models/quicksrnetmedium_quantized/export.py b/qai_hub_models/models/quicksrnetmedium_quantized/export.py index 62f27fb2..18909474 100644 --- a/qai_hub_models/models/quicksrnetmedium_quantized/export.py +++ b/qai_hub_models/models/quicksrnetmedium_quantized/export.py @@ -192,11 +192,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/quicksrnetmedium_quantized/perf.yaml b/qai_hub_models/models/quicksrnetmedium_quantized/perf.yaml index 6ef23d03..f1d881d8 100644 --- a/qai_hub_models/models/quicksrnetmedium_quantized/perf.yaml +++ b/qai_hub_models/models/quicksrnetmedium_quantized/perf.yaml @@ -93,7 +93,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:52.125258Z' + timestamp: '2024-05-29T18:59:42.119455Z' - torchscript_onnx_tflite: inference_time: 812.0 throughput: 1231.527093596059 @@ -146,7 +146,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:52.125294Z' + timestamp: '2024-05-29T18:59:42.119496Z' - torchscript_onnx_tflite: inference_time: 1898.0 throughput: 526.8703898840885 @@ -184,7 +184,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:52.125317Z' + timestamp: '2024-05-29T18:59:42.119523Z' - torchscript_onnx_tflite: inference_time: 2862.0 throughput: 349.4060097833683 @@ -222,7 +222,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs6490 - timestamp: '2024-05-24T06:12:52.125339Z' + timestamp: '2024-05-29T18:59:42.119548Z' - torchscript_onnx_tflite: inference_time: 8787.0 throughput: 113.80448389666553 @@ -245,7 +245,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8250 - timestamp: '2024-05-24T06:12:52.125353Z' + timestamp: '2024-05-29T18:59:42.119564Z' - torchscript_onnx_qnn: inference_time: 750.0 throughput: 1333.3333333333333 @@ -276,21 +276,6 @@ models: total_layers: 16 job_id: j1glk8elp job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 3379.0 - throughput: 295.9455460195324 - estimated_peak_memory_range: - min: 15773696 - max: 15773696 - primary_compute_unit: CPU - precision: fp32 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 17 - total_layers: 17 - job_id: jw561mq7p - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -298,4 +283,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:52.125385Z' + timestamp: '2024-05-29T18:59:42.119590Z' diff --git a/qai_hub_models/models/quicksrnetsmall/export.py b/qai_hub_models/models/quicksrnetsmall/export.py index 95127b72..4d78a4de 100644 --- a/qai_hub_models/models/quicksrnetsmall/export.py +++ b/qai_hub_models/models/quicksrnetsmall/export.py @@ -184,11 +184,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/quicksrnetsmall/perf.yaml b/qai_hub_models/models/quicksrnetsmall/perf.yaml index 7b374eeb..39e979d9 100644 --- a/qai_hub_models/models/quicksrnetsmall/perf.yaml +++ b/qai_hub_models/models/quicksrnetsmall/perf.yaml @@ -87,7 +87,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:52.164500Z' + timestamp: '2024-05-29T18:59:42.162704Z' - torchscript_onnx_tflite: inference_time: 896.0 throughput: 1116.0714285714287 @@ -140,7 +140,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:52.164538Z' + timestamp: '2024-05-29T18:59:42.162746Z' - torchscript_onnx_tflite: inference_time: 1357.0 throughput: 736.9196757553427 @@ -178,7 +178,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:52.164560Z' + timestamp: '2024-05-29T18:59:42.162771Z' - torchscript_onnx_qnn: inference_time: 1147.0 throughput: 871.8395815170009 @@ -209,21 +209,6 @@ models: total_layers: 13 job_id: jvgdv47lg job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 2503.0 - throughput: 399.52057530962844 - estimated_peak_memory_range: - min: 33107968 - max: 33107968 - primary_compute_unit: GPU - precision: fp16 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 1 - layers_on_cpu: 0 - total_layers: 1 - job_id: jz57dn6r5 - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -231,4 +216,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:52.164591Z' + timestamp: '2024-05-29T18:59:42.162795Z' diff --git a/qai_hub_models/models/quicksrnetsmall_quantized/export.py b/qai_hub_models/models/quicksrnetsmall_quantized/export.py index bae9d21d..69aec297 100644 --- a/qai_hub_models/models/quicksrnetsmall_quantized/export.py +++ b/qai_hub_models/models/quicksrnetsmall_quantized/export.py @@ -192,11 +192,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/quicksrnetsmall_quantized/perf.yaml b/qai_hub_models/models/quicksrnetsmall_quantized/perf.yaml index 4c133de3..c846aab6 100644 --- a/qai_hub_models/models/quicksrnetsmall_quantized/perf.yaml +++ b/qai_hub_models/models/quicksrnetsmall_quantized/perf.yaml @@ -93,7 +93,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:52.195182Z' + timestamp: '2024-05-29T18:59:42.196900Z' - torchscript_onnx_tflite: inference_time: 780.0 throughput: 1282.051282051282 @@ -146,7 +146,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:52.195218Z' + timestamp: '2024-05-29T18:59:42.196939Z' - torchscript_onnx_tflite: inference_time: 987.0 throughput: 1013.1712259371834 @@ -184,7 +184,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:52.195241Z' + timestamp: '2024-05-29T18:59:42.196966Z' - torchscript_onnx_tflite: inference_time: 2693.0 throughput: 371.3330857779428 @@ -222,7 +222,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs6490 - timestamp: '2024-05-24T06:12:52.195263Z' + timestamp: '2024-05-29T18:59:42.196990Z' - torchscript_onnx_tflite: inference_time: 6002.0 throughput: 166.61112962345885 @@ -245,7 +245,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8250 - timestamp: '2024-05-24T06:12:52.195277Z' + timestamp: '2024-05-29T18:59:42.197005Z' - torchscript_onnx_qnn: inference_time: 740.0 throughput: 1351.3513513513512 @@ -276,21 +276,6 @@ models: total_layers: 14 job_id: jw561mv7p job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 2958.0 - throughput: 338.0662609871535 - estimated_peak_memory_range: - min: 15798272 - max: 15798272 - primary_compute_unit: CPU - precision: fp32 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 17 - total_layers: 17 - job_id: j1p3m78zg - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -298,4 +283,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:52.195310Z' + timestamp: '2024-05-29T18:59:42.197032Z' diff --git a/qai_hub_models/models/real_esrgan_general_x4v3/export.py b/qai_hub_models/models/real_esrgan_general_x4v3/export.py index 85f0cd5e..588420bb 100644 --- a/qai_hub_models/models/real_esrgan_general_x4v3/export.py +++ b/qai_hub_models/models/real_esrgan_general_x4v3/export.py @@ -184,11 +184,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/real_esrgan_general_x4v3/perf.yaml b/qai_hub_models/models/real_esrgan_general_x4v3/perf.yaml index f6bfaa82..5a0a2c77 100644 --- a/qai_hub_models/models/real_esrgan_general_x4v3/perf.yaml +++ b/qai_hub_models/models/real_esrgan_general_x4v3/perf.yaml @@ -87,7 +87,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:52.234527Z' + timestamp: '2024-05-29T18:59:42.240474Z' - torchscript_onnx_tflite: inference_time: 5502.0 throughput: 181.75209014903672 @@ -140,7 +140,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:52.234577Z' + timestamp: '2024-05-29T18:59:42.240532Z' - torchscript_onnx_tflite: inference_time: 7428.0 throughput: 134.62574044157242 @@ -178,7 +178,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:52.234611Z' + timestamp: '2024-05-29T18:59:42.240569Z' - torchscript_onnx_qnn: inference_time: 9188.0 throughput: 108.837614279495 @@ -209,21 +209,6 @@ models: total_layers: 74 job_id: jmg942kl5 job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 43380.0 - throughput: 23.05209774089442 - estimated_peak_memory_range: - min: 26701824 - max: 26701824 - primary_compute_unit: CPU - precision: fp32 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 70 - total_layers: 70 - job_id: jnp18172g - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -231,4 +216,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:52.234687Z' + timestamp: '2024-05-29T18:59:42.240608Z' diff --git a/qai_hub_models/models/real_esrgan_x4plus/export.py b/qai_hub_models/models/real_esrgan_x4plus/export.py index 0c451bbd..33718c26 100644 --- a/qai_hub_models/models/real_esrgan_x4plus/export.py +++ b/qai_hub_models/models/real_esrgan_x4plus/export.py @@ -169,11 +169,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/real_esrgan_x4plus/perf.yaml b/qai_hub_models/models/real_esrgan_x4plus/perf.yaml index eb7b06fe..4a878b9f 100644 --- a/qai_hub_models/models/real_esrgan_x4plus/perf.yaml +++ b/qai_hub_models/models/real_esrgan_x4plus/perf.yaml @@ -87,7 +87,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:52.265543Z' + timestamp: '2024-05-29T18:59:42.274898Z' - torchscript_onnx_tflite: inference_time: 53158.0 throughput: 18.8118439369427 @@ -140,7 +140,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:52.265866Z' + timestamp: '2024-05-29T18:59:42.275248Z' - torchscript_onnx_tflite: inference_time: 65521.0 throughput: 15.262282321698386 @@ -178,7 +178,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:52.266070Z' + timestamp: '2024-05-29T18:59:42.275481Z' - torchscript_onnx_qnn: inference_time: 73922.0 throughput: 13.527772516977354 @@ -209,21 +209,6 @@ models: total_layers: 1030 job_id: j2p0r9d6p job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 589586.0 - throughput: 1.69610540277415 - estimated_peak_memory_range: - min: 550830080 - max: 550830080 - primary_compute_unit: GPU - precision: fp16 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 1 - layers_on_cpu: 0 - total_layers: 1 - job_id: j1p87r6x5 - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -231,4 +216,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:52.266274Z' + timestamp: '2024-05-29T18:59:42.275721Z' diff --git a/qai_hub_models/models/regnet/export.py b/qai_hub_models/models/regnet/export.py index aab83cf1..ab9a56b7 100644 --- a/qai_hub_models/models/regnet/export.py +++ b/qai_hub_models/models/regnet/export.py @@ -185,11 +185,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/regnet/perf.yaml b/qai_hub_models/models/regnet/perf.yaml index 13cf0882..77a8aad9 100644 --- a/qai_hub_models/models/regnet/perf.yaml +++ b/qai_hub_models/models/regnet/perf.yaml @@ -87,7 +87,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:52.297130Z' + timestamp: '2024-05-29T18:59:42.309964Z' - torchscript_onnx_tflite: inference_time: 1626.0 throughput: 615.0061500615006 @@ -140,7 +140,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:52.297205Z' + timestamp: '2024-05-29T18:59:42.310050Z' - torchscript_onnx_tflite: inference_time: 2328.0 throughput: 429.553264604811 @@ -178,7 +178,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:52.297257Z' + timestamp: '2024-05-29T18:59:42.310103Z' - torchscript_onnx_qnn: inference_time: 2509.0 throughput: 398.5651654045436 @@ -209,21 +209,6 @@ models: total_layers: 190 job_id: jygz7y8kp job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 9503.0 - throughput: 105.2299273913501 - estimated_peak_memory_range: - min: 36933632 - max: 36933632 - primary_compute_unit: GPU - precision: fp16 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 1 - layers_on_cpu: 0 - total_layers: 1 - job_id: jz5w9z16p - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -231,4 +216,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:52.297325Z' + timestamp: '2024-05-29T18:59:42.310167Z' diff --git a/qai_hub_models/models/resnet101/export.py b/qai_hub_models/models/resnet101/export.py index eb2333ff..82072425 100644 --- a/qai_hub_models/models/resnet101/export.py +++ b/qai_hub_models/models/resnet101/export.py @@ -185,11 +185,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/resnet101/perf.yaml b/qai_hub_models/models/resnet101/perf.yaml index 628ad099..1fbddfb5 100644 --- a/qai_hub_models/models/resnet101/perf.yaml +++ b/qai_hub_models/models/resnet101/perf.yaml @@ -87,7 +87,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:52.367330Z' + timestamp: '2024-05-29T18:59:42.388058Z' - torchscript_onnx_tflite: inference_time: 2447.0 throughput: 408.6636697997548 @@ -140,7 +140,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:52.367419Z' + timestamp: '2024-05-29T18:59:42.388157Z' - torchscript_onnx_tflite: inference_time: 3376.0 throughput: 296.2085308056872 @@ -178,7 +178,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:52.367500Z' + timestamp: '2024-05-29T18:59:42.388219Z' - torchscript_onnx_qnn: inference_time: 4026.0 throughput: 248.38549428713364 @@ -209,21 +209,6 @@ models: total_layers: 247 job_id: jz5w9z46p job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 22994.0 - throughput: 43.489605984169785 - estimated_peak_memory_range: - min: 36876288 - max: 36876288 - primary_compute_unit: GPU - precision: fp16 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 1 - layers_on_cpu: 0 - total_layers: 1 - job_id: jmg942dl5 - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -231,4 +216,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:52.367571Z' + timestamp: '2024-05-29T18:59:42.388290Z' diff --git a/qai_hub_models/models/resnet101_quantized/export.py b/qai_hub_models/models/resnet101_quantized/export.py index d1b5cf01..feb84bd3 100644 --- a/qai_hub_models/models/resnet101_quantized/export.py +++ b/qai_hub_models/models/resnet101_quantized/export.py @@ -193,11 +193,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/resnet101_quantized/perf.yaml b/qai_hub_models/models/resnet101_quantized/perf.yaml index 21a5012f..6dea8107 100644 --- a/qai_hub_models/models/resnet101_quantized/perf.yaml +++ b/qai_hub_models/models/resnet101_quantized/perf.yaml @@ -93,7 +93,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:52.397966Z' + timestamp: '2024-05-29T18:59:42.422642Z' - torchscript_onnx_tflite: inference_time: 936.0 throughput: 1068.3760683760684 @@ -146,7 +146,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:52.398038Z' + timestamp: '2024-05-29T18:59:42.422723Z' - torchscript_onnx_tflite: inference_time: 1182.0 throughput: 846.0236886632825 @@ -184,7 +184,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:52.398101Z' + timestamp: '2024-05-29T18:59:42.422775Z' - torchscript_onnx_tflite: inference_time: 4715.0 throughput: 212.08907741251326 @@ -222,7 +222,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs6490 - timestamp: '2024-05-24T06:12:52.398164Z' + timestamp: '2024-05-29T18:59:42.422828Z' - torchscript_onnx_tflite: inference_time: 17085.0 throughput: 58.5308750365818 @@ -245,7 +245,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8250 - timestamp: '2024-05-24T06:12:52.398188Z' + timestamp: '2024-05-29T18:59:42.422856Z' - torchscript_onnx_qnn: inference_time: 1424.0 throughput: 702.2471910112359 @@ -276,21 +276,6 @@ models: total_layers: 154 job_id: j1glk8l8p job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 235061.0 - throughput: 4.25421486337589 - estimated_peak_memory_range: - min: 20107264 - max: 20107264 - primary_compute_unit: GPU - precision: int8 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 1 - layers_on_cpu: 0 - total_layers: 1 - job_id: jw561mw0p - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -298,4 +283,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:52.398249Z' + timestamp: '2024-05-29T18:59:42.422910Z' diff --git a/qai_hub_models/models/resnet18/export.py b/qai_hub_models/models/resnet18/export.py index 33eb0108..0bcf2350 100644 --- a/qai_hub_models/models/resnet18/export.py +++ b/qai_hub_models/models/resnet18/export.py @@ -185,11 +185,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/resnet18/perf.yaml b/qai_hub_models/models/resnet18/perf.yaml index 1174f727..7e1fee47 100644 --- a/qai_hub_models/models/resnet18/perf.yaml +++ b/qai_hub_models/models/resnet18/perf.yaml @@ -87,7 +87,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:52.437513Z' + timestamp: '2024-05-29T18:59:42.466422Z' - torchscript_onnx_tflite: inference_time: 980.0 throughput: 1020.4081632653061 @@ -140,7 +140,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:52.437560Z' + timestamp: '2024-05-29T18:59:42.466474Z' - torchscript_onnx_tflite: inference_time: 1394.0 throughput: 717.3601147776184 @@ -178,7 +178,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:52.437588Z' + timestamp: '2024-05-29T18:59:42.466506Z' - torchscript_onnx_qnn: inference_time: 1561.0 throughput: 640.6149903907751 @@ -209,21 +209,6 @@ models: total_layers: 55 job_id: jz57dn4l5 job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 6003.0 - throughput: 166.58337497917708 - estimated_peak_memory_range: - min: 22487040 - max: 22487040 - primary_compute_unit: CPU - precision: fp32 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 26 - total_layers: 26 - job_id: jqp4w41vg - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -231,4 +216,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:52.437629Z' + timestamp: '2024-05-29T18:59:42.466542Z' diff --git a/qai_hub_models/models/resnet18_quantized/export.py b/qai_hub_models/models/resnet18_quantized/export.py index b1b87e94..22973f71 100644 --- a/qai_hub_models/models/resnet18_quantized/export.py +++ b/qai_hub_models/models/resnet18_quantized/export.py @@ -193,11 +193,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/resnet18_quantized/perf.yaml b/qai_hub_models/models/resnet18_quantized/perf.yaml index d137c363..6c6466e8 100644 --- a/qai_hub_models/models/resnet18_quantized/perf.yaml +++ b/qai_hub_models/models/resnet18_quantized/perf.yaml @@ -93,7 +93,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:52.468196Z' + timestamp: '2024-05-29T18:59:42.500897Z' - torchscript_onnx_tflite: inference_time: 342.0 throughput: 2923.9766081871344 @@ -146,7 +146,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:52.468238Z' + timestamp: '2024-05-29T18:59:42.500945Z' - torchscript_onnx_tflite: inference_time: 438.0 throughput: 2283.10502283105 @@ -184,7 +184,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:52.468264Z' + timestamp: '2024-05-29T18:59:42.500976Z' - torchscript_onnx_tflite: inference_time: 1473.0 throughput: 678.8866259334691 @@ -222,7 +222,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs6490 - timestamp: '2024-05-24T06:12:52.468292Z' + timestamp: '2024-05-29T18:59:42.501006Z' - torchscript_onnx_tflite: inference_time: 7020.0 throughput: 142.45014245014244 @@ -245,7 +245,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8250 - timestamp: '2024-05-24T06:12:52.468307Z' + timestamp: '2024-05-29T18:59:42.501023Z' - torchscript_onnx_qnn: inference_time: 836.0 throughput: 1196.1722488038276 @@ -276,21 +276,6 @@ models: total_layers: 45 job_id: jygz7ymkp job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 1873.0 - throughput: 533.9028296849973 - estimated_peak_memory_range: - min: 708608 - max: 708608 - primary_compute_unit: CPU - precision: fp32 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 47 - total_layers: 47 - job_id: jz5w9zl6p - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -298,4 +283,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:52.468354Z' + timestamp: '2024-05-29T18:59:42.501055Z' diff --git a/qai_hub_models/models/resnet50/export.py b/qai_hub_models/models/resnet50/export.py index 5168cd53..01c180e3 100644 --- a/qai_hub_models/models/resnet50/export.py +++ b/qai_hub_models/models/resnet50/export.py @@ -185,11 +185,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/resnet50/perf.yaml b/qai_hub_models/models/resnet50/perf.yaml index 6f5ff135..8b5c0349 100644 --- a/qai_hub_models/models/resnet50/perf.yaml +++ b/qai_hub_models/models/resnet50/perf.yaml @@ -87,7 +87,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:52.507491Z' + timestamp: '2024-05-29T18:59:42.544373Z' - torchscript_onnx_tflite: inference_time: 1634.0 throughput: 611.9951040391677 @@ -140,7 +140,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:52.507554Z' + timestamp: '2024-05-29T18:59:42.544443Z' - torchscript_onnx_tflite: inference_time: 2281.0 throughput: 438.4042086804033 @@ -178,7 +178,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:52.507592Z' + timestamp: '2024-05-29T18:59:42.544487Z' - torchscript_onnx_qnn: inference_time: 2759.0 throughput: 362.4501631025734 @@ -209,21 +209,6 @@ models: total_layers: 128 job_id: j0px1rl3g job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 15321.0 - throughput: 65.26989099928203 - estimated_peak_memory_range: - min: 37543936 - max: 37543936 - primary_compute_unit: CPU - precision: fp32 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 59 - total_layers: 59 - job_id: jo5mzk0dp - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -231,4 +216,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:52.507648Z' + timestamp: '2024-05-29T18:59:42.544535Z' diff --git a/qai_hub_models/models/resnext101/export.py b/qai_hub_models/models/resnext101/export.py index 3ddcab94..a0347299 100644 --- a/qai_hub_models/models/resnext101/export.py +++ b/qai_hub_models/models/resnext101/export.py @@ -185,11 +185,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/resnext101/perf.yaml b/qai_hub_models/models/resnext101/perf.yaml index 08c71188..751c754f 100644 --- a/qai_hub_models/models/resnext101/perf.yaml +++ b/qai_hub_models/models/resnext101/perf.yaml @@ -87,7 +87,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:52.663235Z' + timestamp: '2024-05-29T18:59:42.693251Z' - torchscript_onnx_tflite: inference_time: 4736.0 throughput: 211.14864864864865 @@ -140,7 +140,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:52.663321Z' + timestamp: '2024-05-29T18:59:42.693354Z' - torchscript_onnx_tflite: inference_time: 6723.0 throughput: 148.74312063067083 @@ -178,7 +178,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:52.663395Z' + timestamp: '2024-05-29T18:59:42.693417Z' - torchscript_onnx_qnn: inference_time: 9099.0 throughput: 109.90218705352237 @@ -209,21 +209,6 @@ models: total_layers: 247 job_id: j1glk88jp job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 61370.0 - throughput: 16.294606485253382 - estimated_peak_memory_range: - min: 85569536 - max: 85569536 - primary_compute_unit: CPU - precision: fp32 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 110 - total_layers: 110 - job_id: jw561mm6p - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -231,4 +216,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:52.663489Z' + timestamp: '2024-05-29T18:59:42.693490Z' diff --git a/qai_hub_models/models/resnext101_quantized/export.py b/qai_hub_models/models/resnext101_quantized/export.py index 04c8fe8c..ac94e2ad 100644 --- a/qai_hub_models/models/resnext101_quantized/export.py +++ b/qai_hub_models/models/resnext101_quantized/export.py @@ -193,11 +193,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/resnext101_quantized/perf.yaml b/qai_hub_models/models/resnext101_quantized/perf.yaml index dda45a30..d14cbbc7 100644 --- a/qai_hub_models/models/resnext101_quantized/perf.yaml +++ b/qai_hub_models/models/resnext101_quantized/perf.yaml @@ -93,7 +93,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:52.694185Z' + timestamp: '2024-05-29T18:59:42.728144Z' - torchscript_onnx_tflite: inference_time: 2099.0 throughput: 476.41734159123394 @@ -146,7 +146,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:52.694254Z' + timestamp: '2024-05-29T18:59:42.728227Z' - torchscript_onnx_tflite: inference_time: 2843.0 throughput: 351.74111853675697 @@ -184,7 +184,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:52.694300Z' + timestamp: '2024-05-29T18:59:42.728279Z' - torchscript_onnx_tflite: inference_time: 10225.0 throughput: 97.79951100244499 @@ -222,7 +222,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs6490 - timestamp: '2024-05-24T06:12:52.694370Z' + timestamp: '2024-05-29T18:59:42.728331Z' - torchscript_onnx_tflite: inference_time: 132850.0 throughput: 7.527286413248024 @@ -245,7 +245,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8250 - timestamp: '2024-05-24T06:12:52.694395Z' + timestamp: '2024-05-29T18:59:42.728360Z' - torchscript_onnx_qnn: inference_time: 3362.0 throughput: 297.441998810232 @@ -276,21 +276,6 @@ models: total_layers: 154 job_id: j0px1rz3g job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 349120.0 - throughput: 2.8643446379468376 - estimated_peak_memory_range: - min: 20082688 - max: 20082688 - primary_compute_unit: GPU - precision: int8 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 1 - layers_on_cpu: 0 - total_layers: 1 - job_id: jo5mzkldp - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -298,4 +283,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:52.694449Z' + timestamp: '2024-05-29T18:59:42.728413Z' diff --git a/qai_hub_models/models/resnext50/export.py b/qai_hub_models/models/resnext50/export.py index d92f338a..b30842ef 100644 --- a/qai_hub_models/models/resnext50/export.py +++ b/qai_hub_models/models/resnext50/export.py @@ -185,11 +185,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/resnext50/perf.yaml b/qai_hub_models/models/resnext50/perf.yaml index a8973ff0..fe0146fc 100644 --- a/qai_hub_models/models/resnext50/perf.yaml +++ b/qai_hub_models/models/resnext50/perf.yaml @@ -87,7 +87,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:52.733705Z' + timestamp: '2024-05-29T18:59:42.771651Z' - torchscript_onnx_tflite: inference_time: 1788.0 throughput: 559.2841163310962 @@ -140,7 +140,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:52.733769Z' + timestamp: '2024-05-29T18:59:42.771721Z' - torchscript_onnx_tflite: inference_time: 2508.0 throughput: 398.72408293460927 @@ -178,7 +178,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:52.733811Z' + timestamp: '2024-05-29T18:59:42.771766Z' - torchscript_onnx_qnn: inference_time: 2906.0 throughput: 344.1156228492774 @@ -209,21 +209,6 @@ models: total_layers: 128 job_id: jw561m06p job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 14752.0 - throughput: 67.78741865509761 - estimated_peak_memory_range: - min: 36884480 - max: 36884480 - primary_compute_unit: GPU - precision: fp16 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 1 - layers_on_cpu: 0 - total_layers: 1 - job_id: j1p3m7r3g - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -231,4 +216,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:52.733877Z' + timestamp: '2024-05-29T18:59:42.771815Z' diff --git a/qai_hub_models/models/resnext50_quantized/export.py b/qai_hub_models/models/resnext50_quantized/export.py index 7f12d261..42967b4b 100644 --- a/qai_hub_models/models/resnext50_quantized/export.py +++ b/qai_hub_models/models/resnext50_quantized/export.py @@ -193,11 +193,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/resnext50_quantized/perf.yaml b/qai_hub_models/models/resnext50_quantized/perf.yaml index d653b647..be21a304 100644 --- a/qai_hub_models/models/resnext50_quantized/perf.yaml +++ b/qai_hub_models/models/resnext50_quantized/perf.yaml @@ -93,7 +93,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:52.764455Z' + timestamp: '2024-05-29T18:59:42.806370Z' - torchscript_onnx_tflite: inference_time: 720.0 throughput: 1388.888888888889 @@ -146,7 +146,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:52.764512Z' + timestamp: '2024-05-29T18:59:42.806430Z' - torchscript_onnx_tflite: inference_time: 939.0 throughput: 1064.9627263045793 @@ -184,7 +184,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:52.764547Z' + timestamp: '2024-05-29T18:59:42.806470Z' - torchscript_onnx_tflite: inference_time: 3073.0 throughput: 325.4149040026033 @@ -222,7 +222,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs6490 - timestamp: '2024-05-24T06:12:52.764588Z' + timestamp: '2024-05-29T18:59:42.806508Z' - torchscript_onnx_tflite: inference_time: 63166.0 throughput: 15.831301649621633 @@ -245,7 +245,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8250 - timestamp: '2024-05-24T06:12:52.764629Z' + timestamp: '2024-05-29T18:59:42.806531Z' - torchscript_onnx_qnn: inference_time: 1357.0 throughput: 736.9196757553427 @@ -276,21 +276,6 @@ models: total_layers: 86 job_id: jvgdv4yzg job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 6285.0 - throughput: 159.10898965791569 - estimated_peak_memory_range: - min: 36773888 - max: 36773888 - primary_compute_unit: CPU - precision: fp32 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 88 - total_layers: 88 - job_id: jz57dn195 - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -298,4 +283,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:52.764682Z' + timestamp: '2024-05-29T18:59:42.806571Z' diff --git a/qai_hub_models/models/sam/export.py b/qai_hub_models/models/sam/export.py index aef175ad..1492c33b 100644 --- a/qai_hub_models/models/sam/export.py +++ b/qai_hub_models/models/sam/export.py @@ -201,11 +201,11 @@ def export_model( # 5. Download the model assets to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) for component_name, compile_job in compile_jobs.items(): diff --git a/qai_hub_models/models/sam/perf.yaml b/qai_hub_models/models/sam/perf.yaml index 57f6b9a6..515dc0f5 100644 --- a/qai_hub_models/models/sam/perf.yaml +++ b/qai_hub_models/models/sam/perf.yaml @@ -50,21 +50,6 @@ models: total_layers: 342 job_id: jqpyd2klp job_status: Passed - torchscript_onnx_ort: - inference_time: 35463.0 - throughput: 28.198403970335278 - estimated_peak_memory_range: - min: 23699456 - max: 67885568 - primary_compute_unit: NPU - precision: fp16 - layer_info: - layers_on_npu: 351 - layers_on_gpu: 0 - layers_on_cpu: 1 - total_layers: 352 - job_id: jw561mxyp - job_status: Passed reference_device_info: name: Samsung Galaxy S23 os: '13' @@ -72,7 +57,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:52.839348Z' + timestamp: '2024-05-29T18:59:42.850316Z' - torchscript_onnx_tflite: inference_time: 35208.0 throughput: 28.402635764598955 @@ -88,21 +73,6 @@ models: total_layers: 342 job_id: j1p87rdo5 job_status: Passed - torchscript_onnx_ort: - inference_time: 25660.0 - throughput: 38.97116134060795 - estimated_peak_memory_range: - min: 25124864 - max: 113827408 - primary_compute_unit: NPU - precision: fp16 - layer_info: - layers_on_npu: 351 - layers_on_gpu: 0 - layers_on_cpu: 1 - total_layers: 352 - job_id: jwgovwxk5 - job_status: Passed reference_device_info: name: Samsung Galaxy S24 os: '14' @@ -110,7 +80,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:52.839443Z' + timestamp: '2024-05-29T18:59:42.850373Z' - torchscript_onnx_tflite: inference_time: 48966.0 throughput: 20.422333864314012 @@ -133,45 +103,15 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:52.839509Z' - - torchscript_onnx_ort: - inference_time: 36087.0 - throughput: 27.71080998697592 - estimated_peak_memory_range: - min: 38932480 - max: 38932480 - primary_compute_unit: NPU - precision: fp16 - layer_info: - layers_on_npu: 351 - layers_on_gpu: 0 - layers_on_cpu: 1 - total_layers: 352 - job_id: j7gjly9ep - job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: jygz7y6xp - job_status: Failed - reference_device_info: + timestamp: '2024-05-29T18:59:42.850426Z' + - reference_device_info: name: Snapdragon X Elite CRD os: '11' form_factor: Compute os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:52.839568Z' + timestamp: '2024-05-29T18:59:42.850433Z' - name: SAMEncoder performance_metrics: - torchscript_onnx_tflite: @@ -189,21 +129,6 @@ models: total_layers: 808 job_id: j2p0r98np job_status: Passed - torchscript_onnx_ort: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: j1p3m7dng - job_status: Failed reference_device_info: name: Samsung Galaxy S23 os: '13' @@ -211,7 +136,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:52.839669Z' + timestamp: '2024-05-29T18:59:42.850531Z' - torchscript_onnx_tflite: inference_time: 10431133.0 throughput: 0.09586686316817167 @@ -227,21 +152,6 @@ models: total_layers: 808 job_id: jogky0wnp job_status: Passed - torchscript_onnx_ort: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: j1pvwm8rg - job_status: Failed reference_device_info: name: Samsung Galaxy S24 os: '14' @@ -249,7 +159,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:52.839762Z' + timestamp: '2024-05-29T18:59:42.850628Z' - torchscript_onnx_tflite: inference_time: 11464171.0 throughput: 0.08722828715656807 @@ -272,42 +182,12 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:52.839848Z' - - torchscript_onnx_ort: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: jlpevxqv5 - job_status: Failed - torchscript_onnx_ort_dml_gpu: - inference_time: 9328223.0 - throughput: 0.10720155382220171 - estimated_peak_memory_range: - min: 3883044864 - max: 3883044864 - primary_compute_unit: CPU - precision: fp32 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 507 - total_layers: 507 - job_id: jz5w9zkmp - job_status: Passed - reference_device_info: + timestamp: '2024-05-29T18:59:42.850725Z' + - reference_device_info: name: Snapdragon X Elite CRD os: '11' form_factor: Compute os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:52.839913Z' + timestamp: '2024-05-29T18:59:42.850731Z' diff --git a/qai_hub_models/models/sesr_m5/export.py b/qai_hub_models/models/sesr_m5/export.py index 8a1c6b62..07e5a42c 100644 --- a/qai_hub_models/models/sesr_m5/export.py +++ b/qai_hub_models/models/sesr_m5/export.py @@ -183,11 +183,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/sesr_m5/perf.yaml b/qai_hub_models/models/sesr_m5/perf.yaml index bcf6bde4..22d99c5a 100644 --- a/qai_hub_models/models/sesr_m5/perf.yaml +++ b/qai_hub_models/models/sesr_m5/perf.yaml @@ -87,7 +87,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:52.881556Z' + timestamp: '2024-05-29T18:59:42.879776Z' - torchscript_onnx_tflite: inference_time: 1621.0 throughput: 616.9031462060457 @@ -140,7 +140,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:52.881597Z' + timestamp: '2024-05-29T18:59:42.879821Z' - torchscript_onnx_tflite: inference_time: 2280.0 throughput: 438.5964912280702 @@ -178,7 +178,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:52.881622Z' + timestamp: '2024-05-29T18:59:42.879850Z' - torchscript_onnx_qnn: inference_time: 2938.0 throughput: 340.3675970047652 @@ -209,21 +209,6 @@ models: total_layers: 33 job_id: jep2md1q5 job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 11997.0 - throughput: 83.3541718763024 - estimated_peak_memory_range: - min: 35692544 - max: 35692544 - primary_compute_unit: CPU - precision: fp32 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 17 - total_layers: 17 - job_id: jqpyd2llp - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -231,4 +216,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:52.881655Z' + timestamp: '2024-05-29T18:59:42.879880Z' diff --git a/qai_hub_models/models/sesr_m5_quantized/export.py b/qai_hub_models/models/sesr_m5_quantized/export.py index 09fd65ea..98ccc38e 100644 --- a/qai_hub_models/models/sesr_m5_quantized/export.py +++ b/qai_hub_models/models/sesr_m5_quantized/export.py @@ -190,11 +190,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/sesr_m5_quantized/perf.yaml b/qai_hub_models/models/sesr_m5_quantized/perf.yaml index 1c0a8a8b..0140a83e 100644 --- a/qai_hub_models/models/sesr_m5_quantized/perf.yaml +++ b/qai_hub_models/models/sesr_m5_quantized/perf.yaml @@ -93,7 +93,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:52.912441Z' + timestamp: '2024-05-29T18:59:42.913920Z' - torchscript_onnx_tflite: inference_time: 1063.0 throughput: 940.7337723424271 @@ -146,7 +146,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:52.912478Z' + timestamp: '2024-05-29T18:59:42.913960Z' - torchscript_onnx_tflite: inference_time: 1331.0 throughput: 751.3148009015778 @@ -184,7 +184,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:52.912501Z' + timestamp: '2024-05-29T18:59:42.913987Z' - torchscript_onnx_tflite: inference_time: 4190.0 throughput: 238.6634844868735 @@ -222,7 +222,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs6490 - timestamp: '2024-05-24T06:12:52.912524Z' + timestamp: '2024-05-29T18:59:42.914013Z' - torchscript_onnx_tflite: inference_time: 5060.0 throughput: 197.62845849802372 @@ -245,7 +245,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8250 - timestamp: '2024-05-24T06:12:52.912540Z' + timestamp: '2024-05-29T18:59:42.914028Z' - torchscript_onnx_qnn: inference_time: 780.0 throughput: 1282.051282051282 @@ -276,21 +276,6 @@ models: total_layers: 19 job_id: jz5w9zjmp job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 3390.0 - throughput: 294.9852507374631 - estimated_peak_memory_range: - min: 33042432 - max: 33042432 - primary_compute_unit: CPU - precision: fp32 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 17 - total_layers: 17 - job_id: jmg942685 - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -298,4 +283,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:52.912570Z' + timestamp: '2024-05-29T18:59:42.914055Z' diff --git a/qai_hub_models/models/shufflenet_v2/export.py b/qai_hub_models/models/shufflenet_v2/export.py index aca822f2..74e70e03 100644 --- a/qai_hub_models/models/shufflenet_v2/export.py +++ b/qai_hub_models/models/shufflenet_v2/export.py @@ -185,11 +185,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/shufflenet_v2/perf.yaml b/qai_hub_models/models/shufflenet_v2/perf.yaml index 2b415ad5..2185322d 100644 --- a/qai_hub_models/models/shufflenet_v2/perf.yaml +++ b/qai_hub_models/models/shufflenet_v2/perf.yaml @@ -87,7 +87,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:52.951878Z' + timestamp: '2024-05-29T18:59:42.957399Z' - torchscript_onnx_tflite: inference_time: 815.0 throughput: 1226.993865030675 @@ -140,7 +140,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:52.951959Z' + timestamp: '2024-05-29T18:59:42.957489Z' - torchscript_onnx_tflite: inference_time: 1229.0 throughput: 813.6696501220505 @@ -178,7 +178,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:52.952010Z' + timestamp: '2024-05-29T18:59:42.957545Z' - torchscript_onnx_qnn: inference_time: 950.0 throughput: 1052.6315789473683 @@ -209,21 +209,6 @@ models: total_layers: 223 job_id: jqpyd29lp job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 1723.0 - throughput: 580.3830528148578 - estimated_peak_memory_range: - min: 12292096 - max: 12292096 - primary_compute_unit: GPU - precision: fp16 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 1 - layers_on_cpu: 0 - total_layers: 1 - job_id: j2p0r9nnp - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -231,4 +216,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:52.952074Z' + timestamp: '2024-05-29T18:59:42.957606Z' diff --git a/qai_hub_models/models/shufflenet_v2_quantized/export.py b/qai_hub_models/models/shufflenet_v2_quantized/export.py index c5e34673..4692513f 100644 --- a/qai_hub_models/models/shufflenet_v2_quantized/export.py +++ b/qai_hub_models/models/shufflenet_v2_quantized/export.py @@ -193,11 +193,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/shufflenet_v2_quantized/perf.yaml b/qai_hub_models/models/shufflenet_v2_quantized/perf.yaml index 15322ccd..0b0a30d7 100644 --- a/qai_hub_models/models/shufflenet_v2_quantized/perf.yaml +++ b/qai_hub_models/models/shufflenet_v2_quantized/perf.yaml @@ -71,21 +71,6 @@ models: total_layers: 122 job_id: j1p3m7kng job_status: Passed - torchscript_onnx_ort: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: jygz7yexp - job_status: Failed reference_device_info: name: Samsung Galaxy S23 os: '13' @@ -93,7 +78,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:52.982555Z' + timestamp: '2024-05-29T18:59:42.991988Z' - torchscript_onnx_tflite: inference_time: 455.0 throughput: 2197.802197802198 @@ -124,21 +109,6 @@ models: total_layers: 122 job_id: jwgovwyk5 job_status: Passed - torchscript_onnx_ort: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: jz5w9zomp - job_status: Failed reference_device_info: name: Samsung Galaxy S24 os: '14' @@ -146,7 +116,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:52.982615Z' + timestamp: '2024-05-29T18:59:42.992051Z' - torchscript_onnx_tflite: inference_time: 623.0 throughput: 1605.1364365971108 @@ -184,7 +154,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:52.982671Z' + timestamp: '2024-05-29T18:59:42.992107Z' - torchscript_onnx_tflite: inference_time: 972.0 throughput: 1028.80658436214 @@ -222,7 +192,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs6490 - timestamp: '2024-05-24T06:12:52.982751Z' + timestamp: '2024-05-29T18:59:42.992163Z' - torchscript_onnx_tflite: inference_time: 10657.0 throughput: 93.83503800319039 @@ -245,7 +215,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8250 - timestamp: '2024-05-24T06:12:52.982784Z' + timestamp: '2024-05-29T18:59:42.992198Z' - torchscript_onnx_qnn: inference_time: 709.0 throughput: 1410.4372355430182 @@ -261,36 +231,6 @@ models: total_layers: 122 job_id: j1pvwm3rg job_status: Passed - torchscript_onnx_ort: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: jmg942v85 - job_status: Failed - torchscript_onnx_ort_dml_gpu: - inference_time: 1501.0 - throughput: 666.2225183211193 - estimated_peak_memory_range: - min: 5361664 - max: 5361664 - primary_compute_unit: CPU - precision: fp32 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 263 - total_layers: 263 - job_id: jnp18107g - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -298,4 +238,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:52.982847Z' + timestamp: '2024-05-29T18:59:42.992226Z' diff --git a/qai_hub_models/models/sinet/export.py b/qai_hub_models/models/sinet/export.py index 621d5c6f..dcfe796d 100644 --- a/qai_hub_models/models/sinet/export.py +++ b/qai_hub_models/models/sinet/export.py @@ -184,11 +184,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/sinet/perf.yaml b/qai_hub_models/models/sinet/perf.yaml index adfdb53a..8a9f923a 100644 --- a/qai_hub_models/models/sinet/perf.yaml +++ b/qai_hub_models/models/sinet/perf.yaml @@ -87,7 +87,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:53.021910Z' + timestamp: '2024-05-29T18:59:43.027778Z' - torchscript_onnx_tflite: inference_time: 1189.0 throughput: 841.0428931875525 @@ -140,7 +140,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:53.021999Z' + timestamp: '2024-05-29T18:59:43.027880Z' - torchscript_onnx_tflite: inference_time: 1802.0 throughput: 554.9389567147614 @@ -178,7 +178,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:53.022060Z' + timestamp: '2024-05-29T18:59:43.027978Z' - torchscript_onnx_qnn: inference_time: 1410.0 throughput: 709.2198581560284 @@ -209,21 +209,6 @@ models: total_layers: 229 job_id: j2p0r9ynp job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 2942.0 - throughput: 339.9048266485384 - estimated_peak_memory_range: - min: 13574144 - max: 13574144 - primary_compute_unit: GPU - precision: fp16 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 1 - layers_on_cpu: 0 - total_layers: 1 - job_id: j1p87roo5 - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -231,4 +216,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:53.022127Z' + timestamp: '2024-05-29T18:59:43.028046Z' diff --git a/qai_hub_models/models/squeezenet1_1/export.py b/qai_hub_models/models/squeezenet1_1/export.py index 2eb125a0..5185e0bc 100644 --- a/qai_hub_models/models/squeezenet1_1/export.py +++ b/qai_hub_models/models/squeezenet1_1/export.py @@ -185,11 +185,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/squeezenet1_1/perf.yaml b/qai_hub_models/models/squeezenet1_1/perf.yaml index cc47e39f..438ef1ef 100644 --- a/qai_hub_models/models/squeezenet1_1/perf.yaml +++ b/qai_hub_models/models/squeezenet1_1/perf.yaml @@ -87,7 +87,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:53.052876Z' + timestamp: '2024-05-29T18:59:43.062080Z' - torchscript_onnx_tflite: inference_time: 462.0 throughput: 2164.5021645021643 @@ -140,7 +140,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:53.052927Z' + timestamp: '2024-05-29T18:59:43.062134Z' - torchscript_onnx_tflite: inference_time: 655.0 throughput: 1526.7175572519084 @@ -178,7 +178,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:53.052956Z' + timestamp: '2024-05-29T18:59:43.062167Z' - torchscript_onnx_qnn: inference_time: 825.0 throughput: 1212.121212121212 @@ -209,21 +209,6 @@ models: total_layers: 71 job_id: jygz7y9xp job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 1650.0 - throughput: 606.060606060606 - estimated_peak_memory_range: - min: 20099072 - max: 20099072 - primary_compute_unit: GPU - precision: fp16 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 1 - layers_on_cpu: 0 - total_layers: 1 - job_id: jz5w9zvmp - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -231,4 +216,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:53.053001Z' + timestamp: '2024-05-29T18:59:43.062205Z' diff --git a/qai_hub_models/models/squeezenet1_1_quantized/export.py b/qai_hub_models/models/squeezenet1_1_quantized/export.py index 2e08f45f..ba6e7087 100644 --- a/qai_hub_models/models/squeezenet1_1_quantized/export.py +++ b/qai_hub_models/models/squeezenet1_1_quantized/export.py @@ -193,11 +193,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/squeezenet1_1_quantized/perf.yaml b/qai_hub_models/models/squeezenet1_1_quantized/perf.yaml index db5e5eaa..19b393c3 100644 --- a/qai_hub_models/models/squeezenet1_1_quantized/perf.yaml +++ b/qai_hub_models/models/squeezenet1_1_quantized/perf.yaml @@ -93,7 +93,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:53.083759Z' + timestamp: '2024-05-29T18:59:43.096271Z' - torchscript_onnx_tflite: inference_time: 180.0 throughput: 5555.555555555556 @@ -146,7 +146,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:53.083808Z' + timestamp: '2024-05-29T18:59:43.096323Z' - torchscript_onnx_tflite: inference_time: 221.0 throughput: 4524.886877828054 @@ -184,7 +184,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:53.083836Z' + timestamp: '2024-05-29T18:59:43.096354Z' - torchscript_onnx_tflite: inference_time: 533.0 throughput: 1876.172607879925 @@ -222,7 +222,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs6490 - timestamp: '2024-05-24T06:12:53.083879Z' + timestamp: '2024-05-29T18:59:43.096386Z' - torchscript_onnx_tflite: inference_time: 4102.0 throughput: 243.78352023403218 @@ -245,7 +245,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8250 - timestamp: '2024-05-24T06:12:53.083899Z' + timestamp: '2024-05-29T18:59:43.096403Z' - torchscript_onnx_qnn: inference_time: 562.0 throughput: 1779.3594306049822 @@ -276,21 +276,6 @@ models: total_layers: 49 job_id: joprydqkg job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 968.0 - throughput: 1033.0578512396694 - estimated_peak_memory_range: - min: 4419584 - max: 4419584 - primary_compute_unit: CPU - precision: fp32 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 51 - total_layers: 51 - job_id: jep2md665 - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -298,4 +283,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:53.083938Z' + timestamp: '2024-05-29T18:59:43.096437Z' diff --git a/qai_hub_models/models/stylegan2/export.py b/qai_hub_models/models/stylegan2/export.py index 8b0c292d..4a1053bc 100644 --- a/qai_hub_models/models/stylegan2/export.py +++ b/qai_hub_models/models/stylegan2/export.py @@ -176,11 +176,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/stylegan2/perf.yaml b/qai_hub_models/models/stylegan2/perf.yaml index 10d81134..b308729f 100644 --- a/qai_hub_models/models/stylegan2/perf.yaml +++ b/qai_hub_models/models/stylegan2/perf.yaml @@ -50,36 +50,6 @@ models: total_layers: 480 job_id: jqpyd2w0p job_status: Passed - torchscript_onnx_qnn: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: jogky0mvp - job_status: Failed - torchscript_onnx_ort: - inference_time: 620107.0 - throughput: 1.6126249179577072 - estimated_peak_memory_range: - min: 199094272 - max: 356043408 - primary_compute_unit: NPU - precision: fp16 - layer_info: - layers_on_npu: 454 - layers_on_gpu: 0 - layers_on_cpu: 89 - total_layers: 543 - job_id: jw561mlnp - job_status: Passed reference_device_info: name: Samsung Galaxy S23 os: '13' @@ -87,7 +57,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:53.123433Z' + timestamp: '2024-05-29T18:59:43.139435Z' - torchscript_onnx_tflite: inference_time: 1246952.0 throughput: 0.8019554882625795 @@ -103,36 +73,6 @@ models: total_layers: 480 job_id: j2p0r970p job_status: Passed - torchscript_onnx_qnn: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: jn5q21oe5 - job_status: Failed - torchscript_onnx_ort: - inference_time: 501195.0 - throughput: 1.9952313969612625 - estimated_peak_memory_range: - min: 307884032 - max: 1078043600 - primary_compute_unit: NPU - precision: fp16 - layer_info: - layers_on_npu: 454 - layers_on_gpu: 0 - layers_on_cpu: 89 - total_layers: 543 - job_id: j1p3m72mg - job_status: Passed reference_device_info: name: Samsung Galaxy S24 os: '14' @@ -140,7 +80,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:53.123579Z' + timestamp: '2024-05-29T18:59:43.139505Z' - torchscript_onnx_tflite: inference_time: 1690139.0 throughput: 0.5916673125701496 @@ -156,21 +96,6 @@ models: total_layers: 480 job_id: j1p87rvq5 job_status: Passed - torchscript_onnx_qnn: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: j1glk8r2p - job_status: Failed reference_device_info: name: QCS8550 (Proxy) os: '12' @@ -178,42 +103,12 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:53.123640Z' - - torchscript_onnx_ort: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: jwgovwq15 - job_status: Failed - torchscript_onnx_ort_dml_gpu: - inference_time: 274732.0 - throughput: 3.639910894981291 - estimated_peak_memory_range: - min: 278179840 - max: 278179840 - primary_compute_unit: CPU - precision: fp32 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 450 - total_layers: 450 - job_id: j1pvwmxzg - job_status: Passed - reference_device_info: + timestamp: '2024-05-29T18:59:43.139572Z' + - reference_device_info: name: Snapdragon X Elite CRD os: '11' form_factor: Compute os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:53.123698Z' + timestamp: '2024-05-29T18:59:43.139580Z' diff --git a/qai_hub_models/models/swin_base/export.py b/qai_hub_models/models/swin_base/export.py index 75622962..a2591912 100644 --- a/qai_hub_models/models/swin_base/export.py +++ b/qai_hub_models/models/swin_base/export.py @@ -185,11 +185,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/swin_base/perf.yaml b/qai_hub_models/models/swin_base/perf.yaml index 2e8c58ee..848eeced 100644 --- a/qai_hub_models/models/swin_base/perf.yaml +++ b/qai_hub_models/models/swin_base/perf.yaml @@ -87,7 +87,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:53.152481Z' + timestamp: '2024-05-29T18:59:43.157642Z' - torchscript_onnx_tflite: inference_time: 26180.0 throughput: 38.19709702062643 @@ -140,7 +140,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:53.152887Z' + timestamp: '2024-05-29T18:59:43.158083Z' - torchscript_onnx_tflite: inference_time: 38498.0 throughput: 25.975375344173724 @@ -178,7 +178,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:53.153148Z' + timestamp: '2024-05-29T18:59:43.158389Z' - torchscript_onnx_qnn: inference_time: 39136.0 throughput: 25.551921504497138 @@ -209,21 +209,6 @@ models: total_layers: 1163 job_id: j0px1r28g job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: jo5mzky7p - job_status: Failed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -231,4 +216,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:53.153380Z' + timestamp: '2024-05-29T18:59:43.158657Z' diff --git a/qai_hub_models/models/swin_small/export.py b/qai_hub_models/models/swin_small/export.py index f682b630..82947e38 100644 --- a/qai_hub_models/models/swin_small/export.py +++ b/qai_hub_models/models/swin_small/export.py @@ -185,11 +185,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/swin_small/perf.yaml b/qai_hub_models/models/swin_small/perf.yaml index a174f8df..3253371f 100644 --- a/qai_hub_models/models/swin_small/perf.yaml +++ b/qai_hub_models/models/swin_small/perf.yaml @@ -87,7 +87,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:53.184335Z' + timestamp: '2024-05-29T18:59:43.193426Z' - torchscript_onnx_tflite: inference_time: 19697.0 throughput: 50.76915266284206 @@ -140,7 +140,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:53.184707Z' + timestamp: '2024-05-29T18:59:43.193885Z' - torchscript_onnx_tflite: inference_time: 29137.0 throughput: 34.320623262518446 @@ -178,7 +178,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:53.185001Z' + timestamp: '2024-05-29T18:59:43.194196Z' - torchscript_onnx_qnn: inference_time: 23958.0 throughput: 41.73971116119876 @@ -209,21 +209,6 @@ models: total_layers: 1158 job_id: j1p3mr0mg job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: jwgov9615 - job_status: Failed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -231,4 +216,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:53.185238Z' + timestamp: '2024-05-29T18:59:43.194465Z' diff --git a/qai_hub_models/models/swin_tiny/export.py b/qai_hub_models/models/swin_tiny/export.py index b9def03f..df0a32e2 100644 --- a/qai_hub_models/models/swin_tiny/export.py +++ b/qai_hub_models/models/swin_tiny/export.py @@ -185,11 +185,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/swin_tiny/perf.yaml b/qai_hub_models/models/swin_tiny/perf.yaml index b289a294..79d0a222 100644 --- a/qai_hub_models/models/swin_tiny/perf.yaml +++ b/qai_hub_models/models/swin_tiny/perf.yaml @@ -87,7 +87,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:53.216439Z' + timestamp: '2024-05-29T18:59:43.229036Z' - torchscript_onnx_tflite: inference_time: 11801.0 throughput: 84.7385814761461 @@ -140,7 +140,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:53.216662Z' + timestamp: '2024-05-29T18:59:43.229264Z' - torchscript_onnx_tflite: inference_time: 17581.0 throughput: 56.87958591661453 @@ -178,7 +178,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:53.216844Z' + timestamp: '2024-05-29T18:59:43.229422Z' - torchscript_onnx_qnn: inference_time: 14657.0 throughput: 68.22678583611926 @@ -209,21 +209,6 @@ models: total_layers: 624 job_id: jo5mzlx7p job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 18013.0 - throughput: 55.51546105590407 - estimated_peak_memory_range: - min: 1421312 - max: 1421312 - primary_compute_unit: CPU - precision: fp32 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 564 - total_layers: 564 - job_id: jegnewvjg - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -231,4 +216,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:53.217033Z' + timestamp: '2024-05-29T18:59:43.229561Z' diff --git a/qai_hub_models/models/trocr/export.py b/qai_hub_models/models/trocr/export.py index c8e8874a..8b74261b 100644 --- a/qai_hub_models/models/trocr/export.py +++ b/qai_hub_models/models/trocr/export.py @@ -191,11 +191,11 @@ def export_model( # 5. Download the model assets to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) for component_name, compile_job in compile_jobs.items(): diff --git a/qai_hub_models/models/trocr/perf.yaml b/qai_hub_models/models/trocr/perf.yaml index a11c8403..521a6a8a 100644 --- a/qai_hub_models/models/trocr/perf.yaml +++ b/qai_hub_models/models/trocr/perf.yaml @@ -50,21 +50,6 @@ models: total_layers: 592 job_id: jopry73kg job_status: Passed - torchscript_onnx_qnn: - inference_time: 122771.0 - throughput: 8.14524602715625 - estimated_peak_memory_range: - min: 1343488 - max: 27644816 - primary_compute_unit: NPU - precision: fp16 - layer_info: - layers_on_npu: 469 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 469 - job_id: jn5q2d6e5 - job_status: Passed torchscript_onnx_ort: inference_time: 111138.0 throughput: 8.997822526948479 @@ -87,7 +72,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:53.247578Z' + timestamp: '2024-05-29T18:59:43.263746Z' - torchscript_onnx_tflite: inference_time: 111585.0 throughput: 8.961778016758524 @@ -103,21 +88,6 @@ models: total_layers: 592 job_id: jqpydy30p job_status: Passed - torchscript_onnx_qnn: - inference_time: 92938.0 - throughput: 10.759861412985 - estimated_peak_memory_range: - min: 1802240 - max: 168859696 - primary_compute_unit: NPU - precision: fp16 - layer_info: - layers_on_npu: 469 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 469 - job_id: jw5610ynp - job_status: Passed torchscript_onnx_ort: inference_time: 84470.0 throughput: 11.838522552385463 @@ -140,7 +110,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:53.247775Z' + timestamp: '2024-05-29T18:59:43.263862Z' - torchscript_onnx_tflite: inference_time: 149520.0 throughput: 6.688068485821295 @@ -156,21 +126,6 @@ models: total_layers: 592 job_id: j1p87kqq5 job_status: Passed - torchscript_onnx_qnn: - inference_time: 123223.0 - throughput: 8.11536807251893 - estimated_peak_memory_range: - min: 1839104 - max: 24876496 - primary_compute_unit: NPU - precision: fp16 - layer_info: - layers_on_npu: 469 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 469 - job_id: jwgov9215 - job_status: Passed reference_device_info: name: QCS8550 (Proxy) os: '12' @@ -178,7 +133,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:53.247883Z' + timestamp: '2024-05-29T18:59:43.263936Z' - torchscript_onnx_ort: inference_time: 111711.0 throughput: 8.951669934026192 @@ -194,21 +149,6 @@ models: total_layers: 396 job_id: jmg94qlm5 job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 53311.0 - throughput: 18.757854851719156 - estimated_peak_memory_range: - min: 2580480 - max: 2580480 - primary_compute_unit: CPU - precision: fp32 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 365 - total_layers: 365 - job_id: jvgdvmx6g - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -216,7 +156,7 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:53.247969Z' + timestamp: '2024-05-29T18:59:43.264003Z' - name: TrOCRDecoder performance_metrics: - torchscript_onnx_tflite: @@ -234,21 +174,6 @@ models: total_layers: 370 job_id: jep2mzy65 job_status: Passed - torchscript_onnx_qnn: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: j1glkqv2p - job_status: Failed torchscript_onnx_ort: inference_time: 2986.0 throughput: 334.8961821835231 @@ -271,7 +196,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:53.248057Z' + timestamp: '2024-05-29T18:59:43.264092Z' - torchscript_onnx_tflite: inference_time: 1972.0 throughput: 507.0993914807302 @@ -287,21 +212,6 @@ models: total_layers: 370 job_id: j2p0rxz0p job_status: Passed - torchscript_onnx_qnn: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: j1p3mrjmg - job_status: Failed torchscript_onnx_ort: inference_time: 2087.0 throughput: 479.1566842357451 @@ -324,7 +234,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:53.248142Z' + timestamp: '2024-05-29T18:59:43.264179Z' - torchscript_onnx_tflite: inference_time: 2738.0 throughput: 365.23009495982467 @@ -340,21 +250,6 @@ models: total_layers: 370 job_id: jogkykevp job_status: Passed - torchscript_onnx_qnn: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: j1pvwn6zg - job_status: Failed reference_device_info: name: QCS8550 (Proxy) os: '12' @@ -362,7 +257,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:53.248191Z' + timestamp: '2024-05-29T18:59:43.264226Z' - torchscript_onnx_ort: inference_time: 2608.0 throughput: 383.4355828220859 @@ -378,21 +273,6 @@ models: total_layers: 353 job_id: jnp18m4ng job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 1469.0 - throughput: 680.7351940095303 - estimated_peak_memory_range: - min: 7159808 - max: 7159808 - primary_compute_unit: CPU - precision: fp32 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 300 - total_layers: 300 - job_id: jz57d8yn5 - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -400,4 +280,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:53.248266Z' + timestamp: '2024-05-29T18:59:43.264271Z' diff --git a/qai_hub_models/models/unet_segmentation/export.py b/qai_hub_models/models/unet_segmentation/export.py index 5e1cd9c2..27cd31cb 100644 --- a/qai_hub_models/models/unet_segmentation/export.py +++ b/qai_hub_models/models/unet_segmentation/export.py @@ -184,11 +184,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/unet_segmentation/perf.yaml b/qai_hub_models/models/unet_segmentation/perf.yaml index a3aceacb..51abe949 100644 --- a/qai_hub_models/models/unet_segmentation/perf.yaml +++ b/qai_hub_models/models/unet_segmentation/perf.yaml @@ -87,7 +87,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:53.302019Z' + timestamp: '2024-05-29T18:59:43.309399Z' - torchscript_onnx_tflite: inference_time: 120918.0 throughput: 8.270067318347971 @@ -140,7 +140,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:53.302079Z' + timestamp: '2024-05-29T18:59:43.309447Z' - torchscript_onnx_tflite: inference_time: 169891.0 throughput: 5.886126987303624 @@ -178,7 +178,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:53.302106Z' + timestamp: '2024-05-29T18:59:43.309477Z' - torchscript_onnx_qnn: inference_time: 190382.0 throughput: 5.252597409418958 @@ -209,21 +209,6 @@ models: total_layers: 53 job_id: jogkykvvp job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: jn5q2d0e5 - job_status: Failed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -231,4 +216,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:53.302149Z' + timestamp: '2024-05-29T18:59:43.309509Z' diff --git a/qai_hub_models/models/vit/export.py b/qai_hub_models/models/vit/export.py index 1e763fed..de6eba4d 100644 --- a/qai_hub_models/models/vit/export.py +++ b/qai_hub_models/models/vit/export.py @@ -187,11 +187,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/vit/perf.yaml b/qai_hub_models/models/vit/perf.yaml index 16488f35..2de2633f 100644 --- a/qai_hub_models/models/vit/perf.yaml +++ b/qai_hub_models/models/vit/perf.yaml @@ -50,21 +50,6 @@ models: total_layers: 535 job_id: j1glkq42p job_status: Passed - torchscript_onnx_qnn: - inference_time: 67155.0 - throughput: 14.890923981833073 - estimated_peak_memory_range: - min: 28672 - max: 45441944 - primary_compute_unit: NPU - precision: fp16 - layer_info: - layers_on_npu: 386 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 386 - job_id: jwgov9z15 - job_status: Passed torchscript_onnx_ort: inference_time: 104122.0 throughput: 9.604118245903843 @@ -87,7 +72,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:53.332981Z' + timestamp: '2024-05-29T18:59:43.343795Z' - torchscript_onnx_tflite: inference_time: 56896.0 throughput: 17.575928008998876 @@ -103,21 +88,6 @@ models: total_layers: 535 job_id: jw56102np job_status: Passed - torchscript_onnx_qnn: - inference_time: 48516.0 - throughput: 20.611756946162092 - estimated_peak_memory_range: - min: 0 - max: 167097360 - primary_compute_unit: NPU - precision: fp16 - layer_info: - layers_on_npu: 386 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 386 - job_id: j1pvwnqzg - job_status: Passed torchscript_onnx_ort: inference_time: 75468.0 throughput: 13.250649281814809 @@ -140,7 +110,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:53.333129Z' + timestamp: '2024-05-29T18:59:43.343903Z' - torchscript_onnx_tflite: inference_time: 79120.0 throughput: 12.639029322548028 @@ -156,21 +126,6 @@ models: total_layers: 535 job_id: j1p3mrnmg job_status: Passed - torchscript_onnx_qnn: - inference_time: 67204.0 - throughput: 14.880066662698649 - estimated_peak_memory_range: - min: 40960 - max: 50583368 - primary_compute_unit: NPU - precision: fp16 - layer_info: - layers_on_npu: 386 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 386 - job_id: jlpevno85 - job_status: Passed reference_device_info: name: QCS8550 (Proxy) os: '12' @@ -178,23 +133,8 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:53.333226Z' - - torchscript_onnx_qnn: - inference_time: 65940.0 - throughput: 15.16530178950561 - estimated_peak_memory_range: - min: 602112 - max: 602112 - primary_compute_unit: NPU - precision: fp16 - layer_info: - layers_on_npu: 385 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 385 - job_id: j7gjl8d1p - job_status: Passed - torchscript_onnx_ort: + timestamp: '2024-05-29T18:59:43.343996Z' + - torchscript_onnx_ort: inference_time: 103336.0 throughput: 9.677169621429124 estimated_peak_memory_range: @@ -209,21 +149,6 @@ models: total_layers: 376 job_id: jmg94q0m5 job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: jnp18m2ng - job_status: Failed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -231,4 +156,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:53.333317Z' + timestamp: '2024-05-29T18:59:43.344048Z' diff --git a/qai_hub_models/models/whisper_base_en/export.py b/qai_hub_models/models/whisper_base_en/export.py index d4cf4a8b..095bc0e3 100644 --- a/qai_hub_models/models/whisper_base_en/export.py +++ b/qai_hub_models/models/whisper_base_en/export.py @@ -191,11 +191,11 @@ def export_model( # 5. Download the model assets to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) for component_name, compile_job in compile_jobs.items(): diff --git a/qai_hub_models/models/whisper_base_en/perf.yaml b/qai_hub_models/models/whisper_base_en/perf.yaml index f47cfead..ecd7e150 100644 --- a/qai_hub_models/models/whisper_base_en/perf.yaml +++ b/qai_hub_models/models/whisper_base_en/perf.yaml @@ -87,7 +87,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:53.364095Z' + timestamp: '2024-05-29T18:59:43.369854Z' - torchscript_onnx_tflite: inference_time: 123349.0 throughput: 8.107078290055046 @@ -140,7 +140,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:53.364265Z' + timestamp: '2024-05-29T18:59:43.370015Z' - torchscript_onnx_tflite: inference_time: 158470.0 throughput: 6.310342651605982 @@ -178,7 +178,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:53.364368Z' + timestamp: '2024-05-29T18:59:43.370125Z' - torchscript_onnx_qnn: inference_time: 463047.0 throughput: 2.159607987958026 @@ -209,21 +209,6 @@ models: total_layers: 380 job_id: jw5610evp job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: jwgov9k45 - job_status: Failed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -231,7 +216,7 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:53.364476Z' + timestamp: '2024-05-29T18:59:43.370231Z' - name: WhisperDecoder performance_metrics: - torchscript_onnx_tflite: @@ -286,7 +271,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:53.364726Z' + timestamp: '2024-05-29T18:59:43.370494Z' - torchscript_onnx_tflite: inference_time: 19456.0 throughput: 51.39802631578947 @@ -339,7 +324,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:53.365013Z' + timestamp: '2024-05-29T18:59:43.370757Z' - torchscript_onnx_tflite: inference_time: 23198.0 throughput: 43.10716441072506 @@ -377,7 +362,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:53.365186Z' + timestamp: '2024-05-29T18:59:43.370934Z' - torchscript_onnx_qnn: inference_time: 13714.0 throughput: 72.91818579553741 @@ -408,21 +393,6 @@ models: total_layers: 844 job_id: j1p3mrvxg job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: j1pvwn07g - job_status: Failed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -430,4 +400,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:53.365350Z' + timestamp: '2024-05-29T18:59:43.371105Z' diff --git a/qai_hub_models/models/whisper_small_en/export.py b/qai_hub_models/models/whisper_small_en/export.py index b4badbfd..dc34702d 100644 --- a/qai_hub_models/models/whisper_small_en/export.py +++ b/qai_hub_models/models/whisper_small_en/export.py @@ -191,11 +191,11 @@ def export_model( # 5. Download the model assets to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) for component_name, compile_job in compile_jobs.items(): diff --git a/qai_hub_models/models/whisper_small_en/perf.yaml b/qai_hub_models/models/whisper_small_en/perf.yaml index 3780472e..d14f6b03 100644 --- a/qai_hub_models/models/whisper_small_en/perf.yaml +++ b/qai_hub_models/models/whisper_small_en/perf.yaml @@ -50,21 +50,6 @@ models: total_layers: 911 job_id: j7gjl8z7p job_status: Passed - torchscript_onnx_qnn: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: jvgdvmekg - job_status: Failed torchscript_onnx_ort: inference_time: 1710031.0 throughput: 0.5847847202770008 @@ -87,7 +72,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:53.423239Z' + timestamp: '2024-05-29T18:59:43.470469Z' - torchscript_onnx_tflite: inference_time: 471828.0 throughput: 2.119416397500784 @@ -140,7 +125,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:53.423582Z' + timestamp: '2024-05-29T18:59:43.470810Z' - torchscript_onnx_tflite: inference_time: 610437.0 throughput: 1.6381706875566193 @@ -156,21 +141,6 @@ models: total_layers: 911 job_id: jmg94qjq5 job_status: Passed - torchscript_onnx_qnn: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: jopry72vg - job_status: Failed reference_device_info: name: QCS8550 (Proxy) os: '12' @@ -178,7 +148,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:53.423678Z' + timestamp: '2024-05-29T18:59:43.470911Z' - torchscript_onnx_qnn: inference_time: 1702121.0 throughput: 0.5875022986027433 @@ -209,21 +179,6 @@ models: total_layers: 884 job_id: jn5q2dr75 job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: jw5610zvp - job_status: Failed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -231,7 +186,7 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:53.423905Z' + timestamp: '2024-05-29T18:59:43.471142Z' - name: WhisperDecoder performance_metrics: - torchscript_onnx_tflite: @@ -286,7 +241,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:53.424559Z' + timestamp: '2024-05-29T18:59:43.471830Z' - torchscript_onnx_tflite: inference_time: 20598.0 throughput: 48.548402757549276 @@ -339,23 +294,8 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:53.425252Z' - - torchscript_onnx_tflite: - inference_time: 27363.0 - throughput: 36.54570039834813 - estimated_peak_memory_range: - min: 16830464 - max: 19976992 - primary_compute_unit: NPU - precision: fp16 - layer_info: - layers_on_npu: 2573 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 2573 - job_id: j1p87qjz5 - job_status: Passed - torchscript_onnx_qnn: + timestamp: '2024-05-29T18:59:43.472545Z' + - torchscript_onnx_qnn: inference_time: 25743.0 throughput: 38.845511401157594 estimated_peak_memory_range: @@ -377,7 +317,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:53.425457Z' + timestamp: '2024-05-29T18:59:43.472758Z' - torchscript_onnx_qnn: inference_time: 20785.0 throughput: 48.11161895597787 @@ -408,21 +348,6 @@ models: total_layers: 2302 job_id: j1glkq2ep job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 50009.0 - throughput: 19.99640064788338 - estimated_peak_memory_range: - min: 127389696 - max: 127389696 - primary_compute_unit: CPU - precision: fp32 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 1759 - total_layers: 1759 - job_id: j1p3mr1xg - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -430,4 +355,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:53.426022Z' + timestamp: '2024-05-29T18:59:43.473190Z' diff --git a/qai_hub_models/models/whisper_tiny_en/export.py b/qai_hub_models/models/whisper_tiny_en/export.py index adf973d2..4c2d1226 100644 --- a/qai_hub_models/models/whisper_tiny_en/export.py +++ b/qai_hub_models/models/whisper_tiny_en/export.py @@ -191,11 +191,11 @@ def export_model( # 5. Download the model assets to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) for component_name, compile_job in compile_jobs.items(): diff --git a/qai_hub_models/models/whisper_tiny_en/perf.yaml b/qai_hub_models/models/whisper_tiny_en/perf.yaml index 842d6292..08635966 100644 --- a/qai_hub_models/models/whisper_tiny_en/perf.yaml +++ b/qai_hub_models/models/whisper_tiny_en/perf.yaml @@ -65,21 +65,6 @@ models: total_layers: 338 job_id: jmg94qyq5 job_status: Passed - torchscript_onnx_ort: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: jopry7evg - job_status: Failed reference_device_info: name: Samsung Galaxy S23 os: '13' @@ -87,7 +72,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:53.484756Z' + timestamp: '2024-05-29T18:59:43.534187Z' - torchscript_onnx_tflite: inference_time: 53307.0 throughput: 18.75926238580299 @@ -118,21 +103,6 @@ models: total_layers: 338 job_id: jvgdvmqkg job_status: Passed - torchscript_onnx_ort: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: jqpydy6rp - job_status: Failed reference_device_info: name: Samsung Galaxy S24 os: '14' @@ -140,7 +110,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:53.484870Z' + timestamp: '2024-05-29T18:59:43.534272Z' - torchscript_onnx_tflite: inference_time: 68587.0 throughput: 14.580022453234578 @@ -178,7 +148,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:53.484948Z' + timestamp: '2024-05-29T18:59:43.534349Z' - torchscript_onnx_qnn: inference_time: 239161.0 throughput: 4.181283737733159 @@ -194,36 +164,6 @@ models: total_layers: 337 job_id: jqp4w2dqg job_status: Passed - torchscript_onnx_ort: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: j1p87kzz5 - job_status: Failed - torchscript_onnx_ort_dml_gpu: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: jn5q2d375 - job_status: Failed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -231,7 +171,7 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:53.485012Z' + timestamp: '2024-05-29T18:59:43.534395Z' - name: WhisperDecoder performance_metrics: - torchscript_onnx_tflite: @@ -286,7 +226,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:53.485169Z' + timestamp: '2024-05-29T18:59:43.534555Z' - torchscript_onnx_tflite: inference_time: 3165.0 throughput: 315.955766192733 @@ -339,7 +279,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:53.485323Z' + timestamp: '2024-05-29T18:59:43.534716Z' - torchscript_onnx_tflite: inference_time: 3880.0 throughput: 257.7319587628866 @@ -377,7 +317,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:53.485430Z' + timestamp: '2024-05-29T18:59:43.534826Z' - torchscript_onnx_qnn: inference_time: 3678.0 throughput: 271.8868950516585 @@ -408,21 +348,6 @@ models: total_layers: 462 job_id: jogkyk3yp job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 3037.0 - throughput: 329.27230819888047 - estimated_peak_memory_range: - min: 21245952 - max: 21245952 - primary_compute_unit: CPU - precision: fp32 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 375 - total_layers: 375 - job_id: j1glkq3ep - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -430,4 +355,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:53.485569Z' + timestamp: '2024-05-29T18:59:43.534928Z' diff --git a/qai_hub_models/models/wideresnet50/export.py b/qai_hub_models/models/wideresnet50/export.py index cc329aa5..5495b5f9 100644 --- a/qai_hub_models/models/wideresnet50/export.py +++ b/qai_hub_models/models/wideresnet50/export.py @@ -185,11 +185,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/wideresnet50/perf.yaml b/qai_hub_models/models/wideresnet50/perf.yaml index 77d53240..75b89214 100644 --- a/qai_hub_models/models/wideresnet50/perf.yaml +++ b/qai_hub_models/models/wideresnet50/perf.yaml @@ -87,7 +87,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:53.544666Z' + timestamp: '2024-05-29T18:59:43.594875Z' - torchscript_onnx_tflite: inference_time: 3633.0 throughput: 275.2546105147261 @@ -140,7 +140,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:53.544731Z' + timestamp: '2024-05-29T18:59:43.594940Z' - torchscript_onnx_tflite: inference_time: 4880.0 throughput: 204.91803278688525 @@ -178,7 +178,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:53.544779Z' + timestamp: '2024-05-29T18:59:43.594981Z' - torchscript_onnx_qnn: inference_time: 5868.0 throughput: 170.41581458759373 @@ -209,21 +209,6 @@ models: total_layers: 128 job_id: jnp18mekg job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 34156.0 - throughput: 29.277432954678535 - estimated_peak_memory_range: - min: 58478592 - max: 58478592 - primary_compute_unit: CPU - precision: fp32 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 59 - total_layers: 59 - job_id: jvgdvmokg - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -231,4 +216,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:53.544859Z' + timestamp: '2024-05-29T18:59:43.595027Z' diff --git a/qai_hub_models/models/wideresnet50_quantized/export.py b/qai_hub_models/models/wideresnet50_quantized/export.py index 568a09d6..26cd34f6 100644 --- a/qai_hub_models/models/wideresnet50_quantized/export.py +++ b/qai_hub_models/models/wideresnet50_quantized/export.py @@ -193,11 +193,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/wideresnet50_quantized/perf.yaml b/qai_hub_models/models/wideresnet50_quantized/perf.yaml index ed068939..4884a99a 100644 --- a/qai_hub_models/models/wideresnet50_quantized/perf.yaml +++ b/qai_hub_models/models/wideresnet50_quantized/perf.yaml @@ -93,7 +93,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:53.575594Z' + timestamp: '2024-05-29T18:59:43.629154Z' - torchscript_onnx_tflite: inference_time: 1382.0 throughput: 723.589001447178 @@ -146,7 +146,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:53.575652Z' + timestamp: '2024-05-29T18:59:43.629211Z' - torchscript_onnx_tflite: inference_time: 1829.0 throughput: 546.7468562055768 @@ -184,7 +184,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:53.575701Z' + timestamp: '2024-05-29T18:59:43.629249Z' - torchscript_onnx_tflite: inference_time: 8003.0 throughput: 124.95314257153568 @@ -222,7 +222,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs6490 - timestamp: '2024-05-24T06:12:53.575752Z' + timestamp: '2024-05-29T18:59:43.629286Z' - torchscript_onnx_tflite: inference_time: 23877.0 throughput: 41.88130837207354 @@ -245,7 +245,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8250 - timestamp: '2024-05-24T06:12:53.575776Z' + timestamp: '2024-05-29T18:59:43.629307Z' - torchscript_onnx_qnn: inference_time: 1947.0 throughput: 513.6106831022086 @@ -276,21 +276,6 @@ models: total_layers: 86 job_id: jw56101vp job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 240031.0 - throughput: 4.166128541730027 - estimated_peak_memory_range: - min: 20582400 - max: 20582400 - primary_compute_unit: GPU - precision: int8 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 1 - layers_on_cpu: 0 - total_layers: 1 - job_id: j1p3mrmxg - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -298,4 +283,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:53.575822Z' + timestamp: '2024-05-29T18:59:43.629346Z' diff --git a/qai_hub_models/models/xlsr/export.py b/qai_hub_models/models/xlsr/export.py index bd27d8fc..9f2e8c9d 100644 --- a/qai_hub_models/models/xlsr/export.py +++ b/qai_hub_models/models/xlsr/export.py @@ -183,11 +183,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/xlsr/perf.yaml b/qai_hub_models/models/xlsr/perf.yaml index b61aad80..90b9cd40 100644 --- a/qai_hub_models/models/xlsr/perf.yaml +++ b/qai_hub_models/models/xlsr/perf.yaml @@ -87,7 +87,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:53.615101Z' + timestamp: '2024-05-29T18:59:43.672571Z' - torchscript_onnx_tflite: inference_time: 1871.0 throughput: 534.4735435595938 @@ -140,7 +140,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:53.615152Z' + timestamp: '2024-05-29T18:59:43.672614Z' - torchscript_onnx_tflite: inference_time: 2515.0 throughput: 397.61431411530816 @@ -178,7 +178,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:53.615180Z' + timestamp: '2024-05-29T18:59:43.672640Z' - torchscript_onnx_qnn: inference_time: 3622.0 throughput: 276.09055770292656 @@ -209,21 +209,6 @@ models: total_layers: 23 job_id: jmg94q4v5 job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 2971.0 - throughput: 336.58700774150117 - estimated_peak_memory_range: - min: 16195584 - max: 16195584 - primary_compute_unit: CPU - precision: fp32 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 14 - total_layers: 14 - job_id: jnp18m8lg - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -231,4 +216,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:53.615216Z' + timestamp: '2024-05-29T18:59:43.672667Z' diff --git a/qai_hub_models/models/xlsr_quantized/export.py b/qai_hub_models/models/xlsr_quantized/export.py index 28fef26b..45d6057b 100644 --- a/qai_hub_models/models/xlsr_quantized/export.py +++ b/qai_hub_models/models/xlsr_quantized/export.py @@ -191,11 +191,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/xlsr_quantized/perf.yaml b/qai_hub_models/models/xlsr_quantized/perf.yaml index b4920ea5..1676715b 100644 --- a/qai_hub_models/models/xlsr_quantized/perf.yaml +++ b/qai_hub_models/models/xlsr_quantized/perf.yaml @@ -93,7 +93,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:53.645808Z' + timestamp: '2024-05-29T18:59:43.706754Z' - torchscript_onnx_tflite: inference_time: 945.0 throughput: 1058.2010582010582 @@ -146,7 +146,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:53.645851Z' + timestamp: '2024-05-29T18:59:43.706797Z' - torchscript_onnx_tflite: inference_time: 1131.0 throughput: 884.1732979664014 @@ -184,7 +184,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:53.645875Z' + timestamp: '2024-05-29T18:59:43.706823Z' - torchscript_onnx_tflite: inference_time: 3650.0 throughput: 273.972602739726 @@ -222,7 +222,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs6490 - timestamp: '2024-05-24T06:12:53.645898Z' + timestamp: '2024-05-29T18:59:43.706849Z' - torchscript_onnx_tflite: inference_time: 14496.0 throughput: 68.98454746136865 @@ -245,7 +245,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8250 - timestamp: '2024-05-24T06:12:53.645930Z' + timestamp: '2024-05-29T18:59:43.706864Z' - torchscript_onnx_qnn: inference_time: 951.0 throughput: 1051.5247108307046 @@ -276,21 +276,6 @@ models: total_layers: 21 job_id: jw561037p job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 3076.0 - throughput: 325.0975292587776 - estimated_peak_memory_range: - min: 15777792 - max: 15777792 - primary_compute_unit: CPU - precision: fp32 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 19 - total_layers: 19 - job_id: j1p3mr4zg - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -298,4 +283,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:53.645982Z' + timestamp: '2024-05-29T18:59:43.706892Z' diff --git a/qai_hub_models/models/yolonas/export.py b/qai_hub_models/models/yolonas/export.py index 335c2d87..15a8dcc5 100644 --- a/qai_hub_models/models/yolonas/export.py +++ b/qai_hub_models/models/yolonas/export.py @@ -183,11 +183,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/yolonas/perf.yaml b/qai_hub_models/models/yolonas/perf.yaml index 79c02db5..c9303e01 100644 --- a/qai_hub_models/models/yolonas/perf.yaml +++ b/qai_hub_models/models/yolonas/perf.yaml @@ -87,7 +87,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:53.685179Z' + timestamp: '2024-05-29T18:59:43.750765Z' - torchscript_onnx_tflite: inference_time: 8112.0 throughput: 123.2741617357002 @@ -140,7 +140,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:53.685293Z' + timestamp: '2024-05-29T18:59:43.750867Z' - torchscript_onnx_tflite: inference_time: 11707.0 throughput: 85.41898009737764 @@ -178,7 +178,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:53.685356Z' + timestamp: '2024-05-29T18:59:43.750934Z' - torchscript_onnx_qnn: inference_time: 11914.0 throughput: 83.9348665435622 @@ -209,21 +209,6 @@ models: total_layers: 290 job_id: jz57d8vr5 job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 48471.0 - throughput: 20.630892698727074 - estimated_peak_memory_range: - min: 70148096 - max: 70148096 - primary_compute_unit: CPU - precision: fp32 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 186 - total_layers: 186 - job_id: jqp4w2jlg - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -231,4 +216,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:53.685454Z' + timestamp: '2024-05-29T18:59:43.751010Z' diff --git a/qai_hub_models/models/yolonas_quantized/export.py b/qai_hub_models/models/yolonas_quantized/export.py index 06023e0e..eadf61e9 100644 --- a/qai_hub_models/models/yolonas_quantized/export.py +++ b/qai_hub_models/models/yolonas_quantized/export.py @@ -191,11 +191,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/yolonas_quantized/perf.yaml b/qai_hub_models/models/yolonas_quantized/perf.yaml index f85a367a..64592233 100644 --- a/qai_hub_models/models/yolonas_quantized/perf.yaml +++ b/qai_hub_models/models/yolonas_quantized/perf.yaml @@ -56,36 +56,6 @@ models: total_layers: 203 job_id: j0px1ze9g job_status: Passed - torchscript_onnx_qnn: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: jqpydyv4p - job_status: Failed - torchscript_onnx_ort: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: jn5q2d9m5 - job_status: Failed reference_device_info: name: Samsung Galaxy S23 os: '13' @@ -93,7 +63,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:53.716280Z' + timestamp: '2024-05-29T18:59:43.785555Z' - torchscript_onnx_tflite: inference_time: 5192.0 throughput: 192.6040061633282 @@ -109,36 +79,6 @@ models: total_layers: 203 job_id: jo5mzlvqp job_status: Passed - torchscript_onnx_qnn: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: j2p0rxeep - job_status: Failed - torchscript_onnx_ort: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: j1glkqelp - job_status: Failed reference_device_info: name: Samsung Galaxy S24 os: '14' @@ -146,7 +86,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:53.716330Z' + timestamp: '2024-05-29T18:59:43.785595Z' - torchscript_onnx_tflite: inference_time: 6975.0 throughput: 143.36917562724014 @@ -162,21 +102,6 @@ models: total_layers: 203 job_id: jegnewrmg job_status: Passed - torchscript_onnx_qnn: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: j1p87kw85 - job_status: Failed reference_device_info: name: QCS8550 (Proxy) os: '12' @@ -184,7 +109,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:53.716382Z' + timestamp: '2024-05-29T18:59:43.785630Z' - torchscript_onnx_tflite: inference_time: 22146.0 throughput: 45.154881242662334 @@ -200,21 +125,6 @@ models: total_layers: 203 job_id: jopry71eg job_status: Passed - torchscript_onnx_qnn: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: jogkykrop - job_status: Failed reference_device_info: name: RB3 Gen 2 (Proxy) os: '12' @@ -222,7 +132,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs6490 - timestamp: '2024-05-24T06:12:53.716422Z' + timestamp: '2024-05-29T18:59:43.785665Z' - torchscript_onnx_tflite: inference_time: 115607.0 throughput: 8.649995242502616 @@ -245,42 +155,12 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8250 - timestamp: '2024-05-24T06:12:53.716451Z' - - torchscript_onnx_ort: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: jw5610q7p - job_status: Failed - torchscript_onnx_ort_dml_gpu: - inference_time: 33786.0 - throughput: 29.5980583673711 - estimated_peak_memory_range: - min: 35999744 - max: 35999744 - primary_compute_unit: CPU - precision: fp32 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 279 - total_layers: 279 - job_id: j1p3mrqzg - job_status: Passed - reference_device_info: + timestamp: '2024-05-29T18:59:43.785698Z' + - reference_device_info: name: Snapdragon X Elite CRD os: '11' form_factor: Compute os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:53.716497Z' + timestamp: '2024-05-29T18:59:43.785705Z' diff --git a/qai_hub_models/models/yolov6/export.py b/qai_hub_models/models/yolov6/export.py index d35e8db0..9895e986 100644 --- a/qai_hub_models/models/yolov6/export.py +++ b/qai_hub_models/models/yolov6/export.py @@ -183,11 +183,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/yolov6/perf.yaml b/qai_hub_models/models/yolov6/perf.yaml index 20d80cd2..6a28fd5e 100644 --- a/qai_hub_models/models/yolov6/perf.yaml +++ b/qai_hub_models/models/yolov6/perf.yaml @@ -87,7 +87,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:53.753310Z' + timestamp: '2024-05-29T18:59:43.810603Z' - torchscript_onnx_tflite: inference_time: 4753.0 throughput: 210.39343572480539 @@ -140,7 +140,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:53.753436Z' + timestamp: '2024-05-29T18:59:43.810694Z' - torchscript_onnx_tflite: inference_time: 6718.0 throughput: 148.85382554331647 @@ -178,7 +178,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:53.753496Z' + timestamp: '2024-05-29T18:59:43.810752Z' - torchscript_onnx_qnn: inference_time: 6760.0 throughput: 147.92899408284023 @@ -209,21 +209,6 @@ models: total_layers: 228 job_id: jz57d86r5 job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 13588.0 - throughput: 73.59434795407712 - estimated_peak_memory_range: - min: 82178048 - max: 82178048 - primary_compute_unit: GPU - precision: fp16 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 1 - layers_on_cpu: 0 - total_layers: 1 - job_id: jqp4w28lg - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -231,4 +216,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:53.753562Z' + timestamp: '2024-05-29T18:59:43.810815Z' diff --git a/qai_hub_models/models/yolov7/export.py b/qai_hub_models/models/yolov7/export.py index 42ecfb2e..a794246c 100644 --- a/qai_hub_models/models/yolov7/export.py +++ b/qai_hub_models/models/yolov7/export.py @@ -183,11 +183,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/yolov7/perf.yaml b/qai_hub_models/models/yolov7/perf.yaml index 1534ec6b..3ea222f6 100644 --- a/qai_hub_models/models/yolov7/perf.yaml +++ b/qai_hub_models/models/yolov7/perf.yaml @@ -50,21 +50,6 @@ models: total_layers: 215 job_id: j0px1zm9g job_status: Passed - torchscript_onnx_qnn: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: jopry79eg - job_status: Failed torchscript_onnx_ort: inference_time: 13428.0 throughput: 74.47125409591898 @@ -87,7 +72,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:53.784331Z' + timestamp: '2024-05-29T18:59:43.845281Z' - torchscript_onnx_tflite: inference_time: 10854.0 throughput: 92.13193292795283 @@ -103,21 +88,6 @@ models: total_layers: 215 job_id: jo5mzl4qp job_status: Passed - torchscript_onnx_qnn: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: jep2mzjm5 - job_status: Failed torchscript_onnx_ort: inference_time: 9618.0 throughput: 103.97171969224371 @@ -140,7 +110,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:53.784403Z' + timestamp: '2024-05-29T18:59:43.845348Z' - torchscript_onnx_tflite: inference_time: 15980.0 throughput: 62.57822277847309 @@ -156,21 +126,6 @@ models: total_layers: 215 job_id: jegnewxmg job_status: Passed - torchscript_onnx_qnn: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: jqpydyn4p - job_status: Failed reference_device_info: name: QCS8550 (Proxy) os: '12' @@ -178,7 +133,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:53.784449Z' + timestamp: '2024-05-29T18:59:43.845383Z' - torchscript_onnx_ort: inference_time: 13426.0 throughput: 74.48234768359899 @@ -194,21 +149,6 @@ models: total_layers: 225 job_id: jogkykdop job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 37595.0 - throughput: 26.599281819390878 - estimated_peak_memory_range: - min: 150376448 - max: 150376448 - primary_compute_unit: GPU - precision: fp16 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 1 - layers_on_cpu: 0 - total_layers: 1 - job_id: jn5q2dwm5 - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -216,4 +156,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:53.784504Z' + timestamp: '2024-05-29T18:59:43.845418Z' diff --git a/qai_hub_models/models/yolov7_quantized/export.py b/qai_hub_models/models/yolov7_quantized/export.py index cfa7c5ad..97078f0f 100644 --- a/qai_hub_models/models/yolov7_quantized/export.py +++ b/qai_hub_models/models/yolov7_quantized/export.py @@ -191,11 +191,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/yolov7_quantized/perf.yaml b/qai_hub_models/models/yolov7_quantized/perf.yaml index bd5fb784..34341b33 100644 --- a/qai_hub_models/models/yolov7_quantized/perf.yaml +++ b/qai_hub_models/models/yolov7_quantized/perf.yaml @@ -56,36 +56,6 @@ models: total_layers: 226 job_id: jw5610v7p job_status: Passed - torchscript_onnx_qnn: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: jz5w9rxjp - job_status: Failed - torchscript_onnx_ort: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: jz57d8kr5 - job_status: Failed reference_device_info: name: Samsung Galaxy S23 os: '13' @@ -93,7 +63,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:53.813034Z' + timestamp: '2024-05-29T18:59:43.871316Z' - torchscript_onnx_tflite: inference_time: 2954.0 throughput: 338.52403520649966 @@ -109,36 +79,6 @@ models: total_layers: 226 job_id: jwgov9md5 job_status: Passed - torchscript_onnx_qnn: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: jmg94q8v5 - job_status: Failed - torchscript_onnx_ort: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: jqp4w2mlg - job_status: Failed reference_device_info: name: Samsung Galaxy S24 os: '14' @@ -146,7 +86,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:53.813097Z' + timestamp: '2024-05-29T18:59:43.871356Z' - torchscript_onnx_tflite: inference_time: 4573.0 throughput: 218.67483052700635 @@ -162,21 +102,6 @@ models: total_layers: 226 job_id: j1pvwn4mg job_status: Passed - torchscript_onnx_qnn: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: jnp18m3lg - job_status: Failed reference_device_info: name: QCS8550 (Proxy) os: '12' @@ -184,7 +109,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:53.813139Z' + timestamp: '2024-05-29T18:59:43.871392Z' - torchscript_onnx_tflite: inference_time: 10865.0 throughput: 92.03865623561896 @@ -200,21 +125,6 @@ models: total_layers: 226 job_id: jlpevn205 job_status: Passed - torchscript_onnx_qnn: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: jvgdvm0lg - job_status: Failed reference_device_info: name: RB3 Gen 2 (Proxy) os: '12' @@ -222,7 +132,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs6490 - timestamp: '2024-05-24T06:12:53.813207Z' + timestamp: '2024-05-29T18:59:43.871427Z' - torchscript_onnx_tflite: inference_time: 92308.0 throughput: 10.833297222342592 @@ -245,42 +155,12 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8250 - timestamp: '2024-05-24T06:12:53.813244Z' - - torchscript_onnx_ort: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: j0px1z39g - job_status: Failed - torchscript_onnx_ort_dml_gpu: - inference_time: 20277.0 - throughput: 49.31696010257928 - estimated_peak_memory_range: - min: 52666368 - max: 52666368 - primary_compute_unit: CPU - precision: fp32 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 256 - total_layers: 256 - job_id: jo5mzloqp - job_status: Passed - reference_device_info: + timestamp: '2024-05-29T18:59:43.871461Z' + - reference_device_info: name: Snapdragon X Elite CRD os: '11' form_factor: Compute os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:53.813292Z' + timestamp: '2024-05-29T18:59:43.871468Z' diff --git a/qai_hub_models/models/yolov8_det/export.py b/qai_hub_models/models/yolov8_det/export.py index 29033c9f..222ace93 100644 --- a/qai_hub_models/models/yolov8_det/export.py +++ b/qai_hub_models/models/yolov8_det/export.py @@ -185,11 +185,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/yolov8_det/perf.yaml b/qai_hub_models/models/yolov8_det/perf.yaml index 8dd83244..6147efd8 100644 --- a/qai_hub_models/models/yolov8_det/perf.yaml +++ b/qai_hub_models/models/yolov8_det/perf.yaml @@ -87,7 +87,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:53.850285Z' + timestamp: '2024-05-29T18:59:43.896469Z' - torchscript_onnx_tflite: inference_time: 4124.0 throughput: 242.48302618816683 @@ -140,7 +140,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:53.850417Z' + timestamp: '2024-05-29T18:59:43.896588Z' - torchscript_onnx_tflite: inference_time: 5889.0 throughput: 169.80811682798438 @@ -178,7 +178,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:53.850492Z' + timestamp: '2024-05-29T18:59:43.896665Z' - torchscript_onnx_qnn: inference_time: 5796.0 throughput: 172.5327812284334 @@ -209,21 +209,6 @@ models: total_layers: 286 job_id: jw5610r7p job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 20244.0 - throughput: 49.39735230191662 - estimated_peak_memory_range: - min: 52838400 - max: 52838400 - primary_compute_unit: CPU - precision: fp32 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 208 - total_layers: 208 - job_id: j1p3mrxzg - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -231,4 +216,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:53.850584Z' + timestamp: '2024-05-29T18:59:43.896740Z' diff --git a/qai_hub_models/models/yolov8_det_quantized/export.py b/qai_hub_models/models/yolov8_det_quantized/export.py index f1247666..8d2d1fa1 100644 --- a/qai_hub_models/models/yolov8_det_quantized/export.py +++ b/qai_hub_models/models/yolov8_det_quantized/export.py @@ -191,11 +191,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/yolov8_det_quantized/perf.yaml b/qai_hub_models/models/yolov8_det_quantized/perf.yaml index db57edf2..ef54d1c0 100644 --- a/qai_hub_models/models/yolov8_det_quantized/perf.yaml +++ b/qai_hub_models/models/yolov8_det_quantized/perf.yaml @@ -56,36 +56,6 @@ models: total_layers: 277 job_id: jwgov9od5 job_status: Passed - torchscript_onnx_qnn: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: jz5w9r8jp - job_status: Failed - torchscript_onnx_ort: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: jmg94qxl5 - job_status: Failed reference_device_info: name: Samsung Galaxy S23 os: '13' @@ -93,7 +63,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:53.881268Z' + timestamp: '2024-05-29T18:59:43.931301Z' - torchscript_onnx_tflite: inference_time: 1597.0 throughput: 626.1740763932373 @@ -109,36 +79,6 @@ models: total_layers: 277 job_id: j1pvwnemg job_status: Passed - torchscript_onnx_qnn: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: jmg94qkv5 - job_status: Failed - torchscript_onnx_ort: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: jnp18mv2g - job_status: Failed reference_device_info: name: Samsung Galaxy S24 os: '14' @@ -146,7 +86,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:53.881338Z' + timestamp: '2024-05-29T18:59:43.931347Z' - torchscript_onnx_tflite: inference_time: 2340.0 throughput: 427.35042735042737 @@ -162,21 +102,6 @@ models: total_layers: 277 job_id: j7gjl8o8p job_status: Passed - torchscript_onnx_qnn: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: jvgdvm8lg - job_status: Failed reference_device_info: name: QCS8550 (Proxy) os: '12' @@ -184,7 +109,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:53.881385Z' + timestamp: '2024-05-29T18:59:43.931388Z' - torchscript_onnx_tflite: inference_time: 7122.0 throughput: 140.40999719180004 @@ -200,21 +125,6 @@ models: total_layers: 277 job_id: jlpevn805 job_status: Passed - torchscript_onnx_qnn: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: jz5w9r16p - job_status: Failed reference_device_info: name: RB3 Gen 2 (Proxy) os: '12' @@ -222,7 +132,7 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs6490 - timestamp: '2024-05-24T06:12:53.881428Z' + timestamp: '2024-05-29T18:59:43.931428Z' - torchscript_onnx_tflite: inference_time: 46687.0 throughput: 21.419238760254462 @@ -245,42 +155,12 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8250 - timestamp: '2024-05-24T06:12:53.881483Z' - - torchscript_onnx_ort: - inference_time: 'null' - throughput: 'null' - estimated_peak_memory_range: - min: 0 - max: 0 - primary_compute_unit: 'null' - precision: 'null' - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 0 - job_id: jvgdvmzeg - job_status: Failed - torchscript_onnx_ort_dml_gpu: - inference_time: 13417.0 - throughput: 74.53230975627935 - estimated_peak_memory_range: - min: 27348992 - max: 27348992 - primary_compute_unit: CPU - precision: fp32 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 370 - total_layers: 370 - job_id: jz57d87l5 - job_status: Passed - reference_device_info: + timestamp: '2024-05-29T18:59:43.931467Z' + - reference_device_info: name: Snapdragon X Elite CRD os: '11' form_factor: Compute os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:53.881537Z' + timestamp: '2024-05-29T18:59:43.931474Z' diff --git a/qai_hub_models/models/yolov8_seg/export.py b/qai_hub_models/models/yolov8_seg/export.py index ac506e78..4156e8c5 100644 --- a/qai_hub_models/models/yolov8_seg/export.py +++ b/qai_hub_models/models/yolov8_seg/export.py @@ -185,11 +185,11 @@ def export_model( # 5. Download the model asset to a local file if not skip_downloading: if target_runtime == TargetRuntime.QNN: - target_runtime_extension = ".so" + target_runtime_extension = "so" elif target_runtime == TargetRuntime.TFLITE: - target_runtime_extension = ".tflite" + target_runtime_extension = "tflite" elif target_runtime == TargetRuntime.ORT: - target_runtime_extension = ".onnx" + target_runtime_extension = "onnx" os.makedirs(output_path, exist_ok=True) target_model: hub.Model = compile_job.get_target_model() # type: ignore diff --git a/qai_hub_models/models/yolov8_seg/perf.yaml b/qai_hub_models/models/yolov8_seg/perf.yaml index 3afc28bc..5b299571 100644 --- a/qai_hub_models/models/yolov8_seg/perf.yaml +++ b/qai_hub_models/models/yolov8_seg/perf.yaml @@ -50,21 +50,6 @@ models: total_layers: 337 job_id: j0px1zd1g job_status: Passed - torchscript_onnx_qnn: - inference_time: 6414.0 - throughput: 155.90894917368257 - estimated_peak_memory_range: - min: 4210688 - max: 14408608 - primary_compute_unit: NPU - precision: fp16 - layer_info: - layers_on_npu: 333 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 333 - job_id: jopry7n9g - job_status: Passed torchscript_onnx_ort: inference_time: 7864.0 throughput: 127.1617497456765 @@ -87,7 +72,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 2 - timestamp: '2024-05-24T06:12:53.918691Z' + timestamp: '2024-05-29T18:59:43.956437Z' - torchscript_onnx_tflite: inference_time: 5327.0 throughput: 187.72292096865027 @@ -103,21 +88,6 @@ models: total_layers: 337 job_id: jo5mzldwp job_status: Passed - torchscript_onnx_qnn: - inference_time: 4564.0 - throughput: 219.10604732690624 - estimated_peak_memory_range: - min: 4931584 - max: 117646032 - primary_compute_unit: NPU - precision: fp16 - layer_info: - layers_on_npu: 333 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 333 - job_id: jep2mzv45 - job_status: Passed torchscript_onnx_ort: inference_time: 5666.0 throughput: 176.49135192375573 @@ -140,7 +110,7 @@ models: os_name: Android manufacturer: Samsung chipset: Snapdragon® 8 Gen 3 - timestamp: '2024-05-24T06:12:53.918832Z' + timestamp: '2024-05-29T18:59:43.956525Z' - torchscript_onnx_tflite: inference_time: 7372.0 throughput: 135.6483993488877 @@ -156,21 +126,6 @@ models: total_layers: 337 job_id: jegnew7rg job_status: Passed - torchscript_onnx_qnn: - inference_time: 6401.0 - throughput: 156.22558975160132 - estimated_peak_memory_range: - min: 4976640 - max: 19548168 - primary_compute_unit: NPU - precision: fp16 - layer_info: - layers_on_npu: 333 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 333 - job_id: j2p0rxv6p - job_status: Passed reference_device_info: name: QCS8550 (Proxy) os: '12' @@ -178,23 +133,8 @@ models: os_name: Android manufacturer: Qualcomm chipset: Qcs8550 - timestamp: '2024-05-24T06:12:53.918909Z' - - torchscript_onnx_qnn: - inference_time: 7574.0 - throughput: 132.03063110641668 - estimated_peak_memory_range: - min: 4923392 - max: 4923392 - primary_compute_unit: NPU - precision: fp16 - layer_info: - layers_on_npu: 333 - layers_on_gpu: 0 - layers_on_cpu: 0 - total_layers: 333 - job_id: jqpydy77p - job_status: Passed - torchscript_onnx_ort: + timestamp: '2024-05-29T18:59:43.956572Z' + - torchscript_onnx_ort: inference_time: 7723.0 throughput: 129.48336138806164 estimated_peak_memory_range: @@ -209,21 +149,6 @@ models: total_layers: 336 job_id: jn5q2dm45 job_status: Passed - torchscript_onnx_ort_dml_gpu: - inference_time: 27209.0 - throughput: 36.75254511374913 - estimated_peak_memory_range: - min: 52838400 - max: 52838400 - primary_compute_unit: CPU - precision: fp32 - layer_info: - layers_on_npu: 0 - layers_on_gpu: 0 - layers_on_cpu: 242 - total_layers: 242 - job_id: j1glkq18p - job_status: Passed reference_device_info: name: Snapdragon X Elite CRD os: '11' @@ -231,4 +156,4 @@ models: os_name: Windows manufacturer: Qualcomm chipset: Snapdragon® X Elite - timestamp: '2024-05-24T06:12:53.919012Z' + timestamp: '2024-05-29T18:59:43.956618Z' diff --git a/qai_hub_models/test/test_async_compile_jobs.py b/qai_hub_models/test/test_async_compile_jobs.py index 21f6f208..18b18235 100644 --- a/qai_hub_models/test/test_async_compile_jobs.py +++ b/qai_hub_models/test/test_async_compile_jobs.py @@ -2,6 +2,7 @@ # Copyright (c) 2024 Qualcomm Innovation Center, Inc. All rights reserved. # SPDX-License-Identifier: BSD-3-Clause # --------------------------------------------------------------------- +import datetime import os import qai_hub as hub @@ -18,10 +19,29 @@ def test_compile_jobs_success(): if os.stat(os.environ["COMPILE_JOBS_FILE"]).st_size == 0: return job_ids = load_yaml(os.environ["COMPILE_JOBS_FILE"]) + failed_jobs = {} + timeout_jobs = {} for name, job_id in job_ids.items(): - result = hub.get_job(job_id).wait() - if not result.success: + job = hub.get_job(job_id) + if job.get_status().running: + # Wait a maximum of 15 minutes for a compile job + timemax = datetime.timedelta(minutes=15) + timediff = datetime.datetime.now() - job.date + if timediff < timemax: + try: + job = job.wait((timemax - timediff).total_seconds()) + except TimeoutError: + timeout_jobs[name] = job_id + else: + timeout_jobs[name] = job_id + elif not job.get_status().success: failed_jobs[name] = job_id + + error_strs = [] if failed_jobs: - raise ValueError(f"The following jobs failed to compile: {failed_jobs}") + error_strs.append(f"The following jobs failed to compile: {failed_jobs}") + if timeout_jobs: + error_strs.append(f"The following jobs timed out: {timeout_jobs}") + if len(error_strs) > 0: + raise ValueError("\n".join(error_strs)) diff --git a/qai_hub_models/utils/scorecard/common.py b/qai_hub_models/utils/scorecard/common.py index b3230a71..c50b66d9 100644 --- a/qai_hub_models/utils/scorecard/common.py +++ b/qai_hub_models/utils/scorecard/common.py @@ -181,6 +181,13 @@ def enabled(self) -> bool: def all_enabled() -> List["ScorecardProfilePath"]: return [x for x in ScorecardProfilePath if x.enabled()] + def include_in_perf_yaml(self) -> bool: + return self in [ + ScorecardProfilePath.QNN, + ScorecardProfilePath.ORT, + ScorecardProfilePath.TFLITE, + ] + @staticmethod def get_parameterized_test_config( aimet_model=False, diff --git a/qai_hub_models/utils/scorecard/job_summary.py b/qai_hub_models/utils/scorecard/job_summary.py index 683dda40..77c2938a 100644 --- a/qai_hub_models/utils/scorecard/job_summary.py +++ b/qai_hub_models/utils/scorecard/job_summary.py @@ -21,11 +21,16 @@ class JobSummary: model_id: str job_id: Optional[str] _device: ScorecardDevice + # Setting for how the JobSummary class should treat a job. + # None | Wait an infinite amount of time the job to finish + # < 0 | Ignore job if running (treat it as skipped) + # >= 0 | Wait this many seconds for the job to finish + max_job_wait_secs: Optional[int] def __post_init__(self): assert self.model_id # Verify Job Exists - if self.job_id: + if self.job_id and (not self.max_job_wait_secs or self.max_job_wait_secs >= 0): assert self.job @classmethod @@ -50,12 +55,16 @@ def job(self) -> Optional[hub.Job]: return None job = hub.get_job(self.job_id) - job.wait() + if job.get_status().running: + if self.max_job_wait_secs and self.max_job_wait_secs < 0: + return None + else: + job.wait(self.max_job_wait_secs) return job @cached_property def skipped(self) -> bool: - return self.job_id is None + return self.job is None @cached_property def failed(self) -> bool: @@ -103,7 +112,10 @@ class CompileJobSummary(JobSummary): @classmethod def from_model_id( - cls: Type["CompileJobSummary"], model_id: str, job_ids: Dict[str, str] + cls: Type["CompileJobSummary"], + model_id: str, + job_ids: Dict[str, str], + max_job_wait_secs=None, ) -> List["CompileJobSummary"]: """ Reads jobs for `model_id` from the dictionary and creates summaries for each. `job_ids` format: @@ -143,6 +155,7 @@ def from_model_id( ), path=path, _device=device, + max_job_wait_secs=max_job_wait_secs, ) ) @@ -167,7 +180,10 @@ class ProfileJobSummary(JobSummary): @classmethod def from_model_id( - cls: Type["ProfileJobSummary"], model_id: str, job_ids: Dict[str, str] + cls: Type["ProfileJobSummary"], + model_id: str, + job_ids: Dict[str, str], + max_job_wait_secs=None, ) -> List["ProfileJobSummary"]: """ Reads jobs for `model_id` from the dictionary and creates summaries for each. `job_ids` format: @@ -208,6 +224,7 @@ def from_model_id( ), _device=device, path=path, + max_job_wait_secs=max_job_wait_secs, ) ) diff --git a/qai_hub_models/utils/scorecard/model_card.py b/qai_hub_models/utils/scorecard/model_card.py index ae5798e4..c812eda1 100644 --- a/qai_hub_models/utils/scorecard/model_card.py +++ b/qai_hub_models/utils/scorecard/model_card.py @@ -9,7 +9,7 @@ import multiprocessing import pprint from dataclasses import dataclass -from typing import Any, Dict, List, Set, Tuple, Union +from typing import Any, Dict, Iterable, List, Set, Tuple, Union import qai_hub as hub @@ -169,10 +169,21 @@ def from_runs(device: ScorecardDevice, path_runs: List[ProfileJobSummary]): return DevicePerfSummary(device, run_per_path) - def get_perf_card(self) -> Dict[str, str | Dict[str, str]]: + def get_perf_card( + self, + include_failed_jobs: bool = True, + exclude_paths: Iterable[ScorecardProfilePath] = [], + ) -> Dict[str, str | Dict[str, str]]: perf_card: Dict[str, str | Dict[str, str]] = {} for path, run in self.run_per_path.items(): - if not run.skipped: # Skipped runs are not included + if ( + not run.skipped # Skipped runs are not included + and path + not in exclude_paths # exclude paths that the user does not want included + and ( + include_failed_jobs or not run.failed + ) # exclude failed jobs if requested + ): perf_card[path.long_name] = run.performance_metrics perf_card["reference_device_info"] = get_reference_device_info(self.device) perf_card["timestamp"] = datetime.datetime.utcnow().isoformat() + "Z" @@ -207,10 +218,14 @@ def from_runs(model_id: str, device_runs: List[ProfileJobSummary]): }, ) - def get_perf_card(self) -> List[Dict[str, Union[str, Dict[str, str]]]]: + def get_perf_card( + self, + include_failed_jobs: bool = True, + exclude_paths: Iterable[ScorecardProfilePath] = [], + ) -> List[Dict[str, Union[str, Dict[str, str]]]]: perf_card = [] for summary in self.runs_per_device.values(): - perf_card.append(summary.get_perf_card()) + perf_card.append(summary.get_perf_card(include_failed_jobs, exclude_paths)) return perf_card def __repr__(self): @@ -223,7 +238,9 @@ class PerfSummary: @staticmethod def from_model_ids( - job_ids: Dict[str, str], model_ids=MODEL_IDS + job_ids: Dict[str, str], + model_ids=MODEL_IDS, + max_job_wait_secs: int | None = None, ) -> Dict[str, PerfSummary]: """ Reads jobs for every `model_id` from the dictionary and creates summaries for each. `job_ids` format: @@ -237,7 +254,12 @@ def from_model_ids( print("Generating Performance Summary for Models") pool = multiprocessing.Pool(processes=15) model_summaries = pool.map( - functools.partial(PerfSummary.from_model_id, job_ids=job_ids), model_ids + functools.partial( + PerfSummary.from_model_id, + job_ids=job_ids, + max_job_wait_secs=max_job_wait_secs, + ), + model_ids, ) pool.close() print("Finished\n") @@ -245,7 +267,9 @@ def from_model_ids( @staticmethod def from_model_id( - model_id: str, job_ids: Dict[str, str] + model_id: str, + job_ids: Dict[str, str], + max_job_wait_secs: int | None = None, ) -> Tuple[str, PerfSummary]: """ Reads jobs for every `model_id` from the dictionary and creates summaries for each. `job_ids` format: @@ -257,7 +281,7 @@ def from_model_id( model_id: List[Summary] """ print(f" {model_id} ") - runs = ProfileJobSummary.from_model_id(model_id, job_ids) + runs = ProfileJobSummary.from_model_id(model_id, job_ids, max_job_wait_secs) return model_id, PerfSummary.from_runs(runs) @staticmethod @@ -284,7 +308,11 @@ def get_chipsets(self) -> Set[str]: ) return chips - def get_perf_card(self) -> Dict[str, str | List[Any] | Dict[str, Any]]: + def get_perf_card( + self, + include_failed_jobs: bool = True, + exclude_paths: Iterable[ScorecardProfilePath] = [], + ) -> Dict[str, str | List[Any] | Dict[str, Any]]: perf_card: Dict[str, str | List[Any] | Dict[str, Any]] = {} chips = self.get_chipsets() @@ -297,7 +325,12 @@ def get_perf_card(self) -> Dict[str, str | List[Any] | Dict[str, Any]]: models_list: List[Dict[str, Any]] = [] for model_id, summary in self.runs_per_model.items(): models_list.append( - {"name": model_id, "performance_metrics": summary.get_perf_card()} + { + "name": model_id, + "performance_metrics": summary.get_perf_card( + include_failed_jobs, exclude_paths + ), + } ) perf_card["models"] = models_list return perf_card @@ -352,7 +385,9 @@ class CompileSummary: @staticmethod def from_model_ids( - job_ids: Dict[str, str], model_ids=MODEL_IDS + job_ids: Dict[str, str], + model_ids=MODEL_IDS, + max_job_wait_secs: int | None = None, ) -> Dict[str, CompileSummary]: """ Reads jobs for every `model_id` from the dictionary and creates summaries for each. `job_ids` format: @@ -368,7 +403,12 @@ def from_model_ids( print("Generating Compilation Summary for Models") pool = multiprocessing.Pool(processes=15) model_summaries = pool.map( - functools.partial(CompileSummary.from_model_id, job_ids=job_ids), model_ids + functools.partial( + CompileSummary.from_model_id, + job_ids=job_ids, + max_job_wait_secs=max_job_wait_secs, + ), + model_ids, ) pool.close() print("Finished\n") @@ -376,7 +416,9 @@ def from_model_ids( @staticmethod def from_model_id( - model_id: str, job_ids: Dict[str, str] + model_id: str, + job_ids: Dict[str, str], + max_job_wait_secs: int | None = None, ) -> Tuple[str, CompileSummary]: """ Reads jobs for every `model_id` from the dictionary and creates summaries for each. `job_ids` format: @@ -390,7 +432,7 @@ def from_model_id( model_id: List[Summary] """ print(f" {model_id} ") - runs = CompileJobSummary.from_model_id(model_id, job_ids) + runs = CompileJobSummary.from_model_id(model_id, job_ids, max_job_wait_secs) return model_id, CompileSummary.from_runs(runs) @staticmethod