From f75187f0e47602fd2692747afdf53f2297a31b89 Mon Sep 17 00:00:00 2001 From: Pradeep Chandrasekaran <160489301+chandrasekaranpradeep@users.noreply.github.com> Date: Mon, 17 Feb 2025 19:19:17 +0530 Subject: [PATCH] Add timeout in subprocess run method for generate and export unique ops config (#1231) The [generate model ops test pipeline](https://github.com/tenstorrent/tt-forge-fe/actions/runs/13328380954/job/37226649520) is currently freezing during the unique ops configuration extraction phase Error: `Failed on "DecomposeEinsum" TVM callback` This error is encountered in the test case: `forge/test/models/pytorch/vision/detr/test_detr.py::test_detr_segmentation[facebook/detr-resnet-50-panoptic]` test cases. To prevent the extraction process from hanging indefinitely, a timeout of 1200 seconds (20 minutes) has been added. This ensures that if the unique ops configuration extraction takes too long, the test will be terminated. --- scripts/model_analysis/unique_ops_utils.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/scripts/model_analysis/unique_ops_utils.py b/scripts/model_analysis/unique_ops_utils.py index 07c8f1f29..2ed94290b 100644 --- a/scripts/model_analysis/unique_ops_utils.py +++ b/scripts/model_analysis/unique_ops_utils.py @@ -16,7 +16,10 @@ def generate_and_export_unique_ops_tests( - test_directory_or_file_path: str, unique_ops_output_directory_path: str, extract_tvm_unique_ops_config: bool = False + test_directory_or_file_path: str, + unique_ops_output_directory_path: str, + extract_tvm_unique_ops_config: bool = False, + timeout: int = 1200, ): """ Collect all the tests that doesn't contain skip_model_analysis marker in the test_directory_or_file_path specified by the user @@ -52,6 +55,7 @@ def generate_and_export_unique_ops_tests( capture_output=True, text=True, check=True, + timeout=timeout, env=dict( os.environ, FORGE_TVM_GENERATE_UNIQUE_OPS_TESTS="1" if not extract_tvm_unique_ops_config else "0", @@ -69,6 +73,12 @@ def generate_and_export_unique_ops_tests( except subprocess.CalledProcessError as e: logger.error(f"Error while running the pytest:\n {e.output}") + except subprocess.TimeoutExpired as e: + logger.error(f"Test timed out after {timeout} seconds") + + except Exception as e: + logger.error(f"An unexpected error occurred while running {test}: {e}") + return model_output_dir_paths