From 95bb1a0ad3004168edf8a4ea91f5a774925f652f Mon Sep 17 00:00:00 2001 From: Vladimir Brkic Date: Mon, 13 Jan 2025 15:02:58 +0000 Subject: [PATCH] Log test vector properties in the test report --- forge/test/operators/pytorch/conftest.py | 43 +++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/forge/test/operators/pytorch/conftest.py b/forge/test/operators/pytorch/conftest.py index 58ab3fec6..dd012db3f 100644 --- a/forge/test/operators/pytorch/conftest.py +++ b/forge/test/operators/pytorch/conftest.py @@ -14,12 +14,17 @@ from test.operators.utils import PyTestUtils from test.operators.utils import FailingReasonsValidation +from ..utils import TestPlanUtils + @pytest.hookimpl(hookwrapper=True) def pytest_runtest_makereport(item: _pytest.python.Function, call: _pytest.runner.CallInfo): outcome: pluggy.callers._Result = yield report: _pytest.reports.TestReport = outcome.get_result() + if report.when == "call" or (report.when == "setup" and report.skipped): + xfail_reason = PyTestUtils.get_xfail_reason(item) + # This hook function is called after each step of the test execution (setup, call, teardown) if call.when == "call": # 'call' is a phase when the test is actually executed @@ -30,7 +35,7 @@ def pytest_runtest_makereport(item: _pytest.python.Function, call: _pytest.runne ) exception_value = call.excinfo.value - xfail_reason = PyTestUtils.get_xfail_reason(item) + if xfail_reason is not None: # an xfail reason is defined for the test valid_reason = FailingReasonsValidation.validate_exception(exception_value, xfail_reason) @@ -50,3 +55,39 @@ def pytest_runtest_makereport(item: _pytest.python.Function, call: _pytest.runne outcome.force_result(new_report) else: logger.debug(f"Test '{item.name}' failed with exception: {type(exception_value)} '{exception_value}'") + + if report.when == "call" or (report.when == "setup" and report.skipped): + try: + log_test_vector_properties(item, report, xfail_reason) + except Exception as e: + logger.error(f"Failed to log test vector properties: {e}") + logger.exception(e) + pass + + +def log_test_vector_properties(item: _pytest.python.Function, report: _pytest.reports.TestReport, xfail_reason: str): + original_name = item.originalname + test_id = item.name + test_id = test_id.replace(f"{original_name}[", "") + test_id = test_id.replace("]", "") + if test_id == "no_device-test_vector0": + # This is not a valid test id. It happens when no tests are selected to run. + return + test_vector = TestPlanUtils.test_id_to_test_vector(test_id) + + item.user_properties.append(("id", test_id)) + item.user_properties.append(("operator", test_vector.operator)) + item.user_properties.append( + ("input_source", test_vector.input_source.name if test_vector.input_source is not None else None) + ) + item.user_properties.append( + ("dev_data_format", test_vector.dev_data_format.name if test_vector.dev_data_format is not None else None) + ) + item.user_properties.append( + ("math_fidelity", test_vector.math_fidelity.name if test_vector.math_fidelity is not None else None) + ) + item.user_properties.append(("input_shape", test_vector.input_shape)) + item.user_properties.append(("kwargs", test_vector.kwargs)) + if xfail_reason is not None: + item.user_properties.append(("xfail_reason", xfail_reason)) + item.user_properties.append(("outcome", report.outcome))