From 67397303d8b35e0d77649781f5bc2a5ecdf9f14a Mon Sep 17 00:00:00 2001 From: Nigel Huang Date: Fri, 28 Feb 2025 08:19:11 +0000 Subject: [PATCH] #0: increase perf bounds --- models/demos/bert_tiny/tests/test_performance.py | 2 +- .../demos/distilbert/tests/test_perf_distilbert.py | 2 +- .../falcon7b_common/tests/run_falcon_end_to_end.py | 2 +- .../falcon7b_common/tests/test_falcon_device_perf.py | 12 ++++++------ .../demos/roberta/tests/test_perf_device_roberta.py | 2 +- .../tests/test_perf_device_squeezebert.py | 2 +- .../resnet50/tests/test_perf_device_resnet50.py | 2 +- .../wormhole/stable_diffusion/tests/test_perf.py | 2 +- models/demos/yolov4/tests/test_perf_yolo.py | 2 +- 9 files changed, 14 insertions(+), 14 deletions(-) diff --git a/models/demos/bert_tiny/tests/test_performance.py b/models/demos/bert_tiny/tests/test_performance.py index 9ce0d3761f8..4ac277bcab2 100644 --- a/models/demos/bert_tiny/tests/test_performance.py +++ b/models/demos/bert_tiny/tests/test_performance.py @@ -119,7 +119,7 @@ def test_perf_device_bare_metal(batch_size, expected_perf): margin = 0.03 if is_wormhole_b0(): - expected_perf = 4439.2 + expected_perf = 4475.2 else: expected_perf = 3460.0 diff --git a/models/demos/distilbert/tests/test_perf_distilbert.py b/models/demos/distilbert/tests/test_perf_distilbert.py index e675163118e..297e6c67d9e 100644 --- a/models/demos/distilbert/tests/test_perf_distilbert.py +++ b/models/demos/distilbert/tests/test_perf_distilbert.py @@ -154,7 +154,7 @@ def test_distilbert_perf_device(batch_size, test, reset_seeds): if is_grayskull(): expected_perf = 57.3 elif is_wormhole_b0(): - expected_perf = 95.5 + expected_perf = 97.0 command = f"pytest tests/ttnn/integration_tests/distilbert/test_ttnn_distilbert.py::test_distilbert_for_question_answering[sequence_size=768-batch_size=8-model_name=distilbert-base-uncased-distilled-squad]" cols = ["DEVICE FW", "DEVICE KERNEL", "DEVICE BRISC KERNEL"] diff --git a/models/demos/falcon7b_common/tests/run_falcon_end_to_end.py b/models/demos/falcon7b_common/tests/run_falcon_end_to_end.py index 700db2908d3..e68ca07a584 100644 --- a/models/demos/falcon7b_common/tests/run_falcon_end_to_end.py +++ b/models/demos/falcon7b_common/tests/run_falcon_end_to_end.py @@ -96,7 +96,7 @@ class DeviceSetup(Enum): DeviceSetup.WORMHOLE_B0: { "BFLOAT16-DRAM": {128: (0.89, 0.92, 0.91), 1024: (0.92, 0.94, 0.95), 2047: (0.95, 0.96, 0.97)}, "BFLOAT16-L1": {128: (0.89, 0.92, 0.91), 1024: (0.92, 0.94, 0.95), 2047: (0.95, 0.96, 0.97)}, - "BFLOAT16-L1_SHARDED": {128: (0.90, 0.91, 0.91), 1024: (0.93, 0.94, 0.96), 2047: (0.92, 0.93, 0.94)}, + "BFLOAT16-L1_SHARDED": {128: (0.89, 0.91, 0.91), 1024: (0.93, 0.94, 0.96), 2047: (0.92, 0.93, 0.94)}, }, DeviceSetup.T3000: { "BFLOAT16-L1_SHARDED": {128: (0.85, 0.89, 0.90), 1024: (0.90, 0.92, 0.93), 2047: (0.95, 0.91, 0.89)} diff --git a/models/demos/falcon7b_common/tests/test_falcon_device_perf.py b/models/demos/falcon7b_common/tests/test_falcon_device_perf.py index 450ed2ba41d..9c34693ed46 100644 --- a/models/demos/falcon7b_common/tests/test_falcon_device_perf.py +++ b/models/demos/falcon7b_common/tests/test_falcon_device_perf.py @@ -90,12 +90,12 @@ def test_device_perf_wh_bare_metal( @pytest.mark.parametrize( "llm_mode, batch, seq_len, kv_cache_len, model_config_str, samples", ( - ("prefill", 1, 128, 0, "BFLOAT16-DRAM", 2060), - ("prefill", 1, 1024, 0, "BFLOAT16-DRAM", 2890), - ("prefill", 1, 2048, 0, "BFLOAT16-DRAM", 2680), - ("decode", 32, 1, 128, "BFLOAT16-L1_SHARDED", 625), - ("decode", 32, 1, 1024, "BFLOAT16-L1_SHARDED", 568), - ("decode", 32, 1, 2047, "BFLOAT16-L1_SHARDED", 550), + ("prefill", 1, 128, 0, "BFLOAT16-DRAM", 2068), + ("prefill", 1, 1024, 0, "BFLOAT16-DRAM", 2895), + ("prefill", 1, 2048, 0, "BFLOAT16-DRAM", 2684), + ("decode", 32, 1, 128, "BFLOAT16-L1_SHARDED", 629), + ("decode", 32, 1, 1024, "BFLOAT16-L1_SHARDED", 572), + ("decode", 32, 1, 2047, "BFLOAT16-L1_SHARDED", 553), ), ) @skip_for_grayskull() diff --git a/models/demos/roberta/tests/test_perf_device_roberta.py b/models/demos/roberta/tests/test_perf_device_roberta.py index 048ee2811a6..71c89be1cea 100644 --- a/models/demos/roberta/tests/test_perf_device_roberta.py +++ b/models/demos/roberta/tests/test_perf_device_roberta.py @@ -18,7 +18,7 @@ def test_perf_device_bare_metal(batch_size, test): subdir = "ttnn_roberta" num_iterations = 1 margin = 0.03 - expected_perf = 154.94 if is_grayskull() else 153.70 + expected_perf = 154.94 if is_grayskull() else 153.9 command = f"pytest tests/ttnn/integration_tests/roberta/test_ttnn_optimized_roberta.py::test_roberta_for_question_answering[{test}]" cols = ["DEVICE FW", "DEVICE KERNEL", "DEVICE BRISC KERNEL"] diff --git a/models/demos/squeezebert/tests/test_perf_device_squeezebert.py b/models/demos/squeezebert/tests/test_perf_device_squeezebert.py index 381bc9f0c9d..630a4d6681d 100644 --- a/models/demos/squeezebert/tests/test_perf_device_squeezebert.py +++ b/models/demos/squeezebert/tests/test_perf_device_squeezebert.py @@ -18,7 +18,7 @@ def test_perf_device_bare_metal(batch_size, test): subdir = "ttnn_squeezebert" num_iterations = 1 margin = 0.03 - expected_perf = 102.7 if is_grayskull() else 298.7 + expected_perf = 102.7 if is_grayskull() else 299.7 command = f"pytest tests/ttnn/integration_tests/squeezebert/test_ttnn_squeezebert.py::test_squeezebert_for_question_answering" cols = ["DEVICE FW", "DEVICE KERNEL", "DEVICE BRISC KERNEL"] diff --git a/models/demos/wormhole/resnet50/tests/test_perf_device_resnet50.py b/models/demos/wormhole/resnet50/tests/test_perf_device_resnet50.py index 3bd90e3d210..bb47da82c48 100644 --- a/models/demos/wormhole/resnet50/tests/test_perf_device_resnet50.py +++ b/models/demos/wormhole/resnet50/tests/test_perf_device_resnet50.py @@ -12,7 +12,7 @@ @pytest.mark.parametrize( "batch_size, test, expected_perf", [ - [16, "16-act_dtype0-weight_dtype0-math_fidelity0-device_params0", 5255], + [16, "16-act_dtype0-weight_dtype0-math_fidelity0-device_params0", 5311.0], ], ) def test_perf_device(batch_size, test, expected_perf): diff --git a/models/demos/wormhole/stable_diffusion/tests/test_perf.py b/models/demos/wormhole/stable_diffusion/tests/test_perf.py index 2056f5efcba..43d1a202a02 100644 --- a/models/demos/wormhole/stable_diffusion/tests/test_perf.py +++ b/models/demos/wormhole/stable_diffusion/tests/test_perf.py @@ -207,7 +207,7 @@ def test_stable_diffusion_perf(device, batch_size, num_inference_steps, expected @pytest.mark.models_device_performance_bare_metal @pytest.mark.parametrize( "expected_kernel_samples_per_second", - ((9.5),), + ((9.8),), ) def test_stable_diffusion_device_perf(expected_kernel_samples_per_second): subdir = "ttnn_stable_diffusion" diff --git a/models/demos/yolov4/tests/test_perf_yolo.py b/models/demos/yolov4/tests/test_perf_yolo.py index caa2652f2f7..8137e0a333d 100644 --- a/models/demos/yolov4/tests/test_perf_yolo.py +++ b/models/demos/yolov4/tests/test_perf_yolo.py @@ -125,7 +125,7 @@ def test_perf_device_bare_metal_yolov4(batch_size, model_name): num_iterations = 1 margin = 0.03 - expected_perf = 51 + expected_perf = 54 command = f"pytest tests/ttnn/integration_tests/yolov4/test_ttnn_yolov4.py" cols = ["DEVICE FW", "DEVICE KERNEL", "DEVICE BRISC KERNEL"]