From 6fe814bfaf663730ba5de1719a1641e12f2369ac Mon Sep 17 00:00:00 2001 From: pmanoj Date: Mon, 12 Feb 2024 10:36:13 +0530 Subject: [PATCH] Propogate the system error to RH (#2296) * fail the run for error propogation * adding missing pkgs --- .../src/azureml/model/mgmt/utils/exceptions.py | 8 ++++++++ assets/training/model_management/tests/dev_conda_env.yaml | 1 + 2 files changed, 9 insertions(+) diff --git a/assets/training/model_management/src/azureml/model/mgmt/utils/exceptions.py b/assets/training/model_management/src/azureml/model/mgmt/utils/exceptions.py index 0e020aaf48..44013aa8df 100644 --- a/assets/training/model_management/src/azureml/model/mgmt/utils/exceptions.py +++ b/assets/training/model_management/src/azureml/model/mgmt/utils/exceptions.py @@ -10,6 +10,9 @@ from azureml._common._error_definition.azureml_error import AzureMLError # type: ignore from azureml._common._error_definition.system_error import ClientError # type: ignore +from azureml.core.run import Run # type: ignore +from azureml.automl.core._run import run_lifecycle_utilities + class ModelImportErrorStrings: """Error strings.""" @@ -207,6 +210,11 @@ def wrapper(*args, **kwargs): logger.error("Exception {} when calling {}".format(azureml_exception, func.__name__)) for handler in logger.handlers: handler.flush() + + # fail the run + run = Run.get_context() + run_lifecycle_utilities.fail_run(run, azureml_exception, is_aml_compute=True) + raise azureml_exception finally: time.sleep(60) # Let telemetry logger flush its logs before terminating. diff --git a/assets/training/model_management/tests/dev_conda_env.yaml b/assets/training/model_management/tests/dev_conda_env.yaml index bf8c53da8a..515a87d30c 100644 --- a/assets/training/model_management/tests/dev_conda_env.yaml +++ b/assets/training/model_management/tests/dev_conda_env.yaml @@ -11,6 +11,7 @@ dependencies: - azureml-mlflow~=1.51.0 - azure-ai-ml~=1.8.0 - azureml-core~=1.52.0 + - azureml-automl-core~=1.52.0 - azureml-telemetry~=1.52.0 - azureml-evaluate-mlflow~=0.0.19 - langcodes