Bhavanatumma/ta2614748

assets/training/model_management/components/convert_model_to_mlflow/spec.yaml

@@ -1,15 +1,15 @@
 $schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json
 
 name: convert_model_to_mlflow
-version: 0.0.15
+version: 0.0.16
 type: command
 
 is_deterministic: True
 
 display_name: Convert models to MLflow
 description: Component converts models from supported frameworks to MLflow model packaging format
 
-environment: azureml://registries/azureml/environments/model-management/versions/13
+environment: azureml://registries/azureml/environments/model-management/versions/15
 
 code: ../../src/
 command: >

assets/training/model_management/components/import_model/spec.yaml

@@ -4,7 +4,7 @@ type: pipeline
 name: import_model
 display_name: Import model
 description: Import a model into a workspace or a registry
-version: 0.0.17
+version: 0.0.18
 
 # Pipeline inputs
 inputs:
@@ -227,7 +227,7 @@ jobs:
         type: uri_folder
 
   convert_model_to_mlflow:
-    component: azureml:convert_model_to_mlflow:0.0.14
+    component: azureml:convert_model_to_mlflow:0.0.16
     compute: ${{parent.inputs.compute}}
     resources:
       instance_type: '${{parent.inputs.instance_type}}'

assets/training/model_management/src/azureml/model/mgmt/processors/transformers/convertors.py

@@ -2,12 +2,15 @@
 # Licensed under the MIT License.
 
 """HFTransformers MLflow model convertors."""
-
+from typing import List
 import transformers
+import platform
+import mlflow
 import os
 import yaml
 from abc import ABC, abstractmethod
 from azureml.evaluate import mlflow as hf_mlflow
+from azureml.core.conda_dependencies import CondaDependencies
 from azureml.model.mgmt.processors.convertors import MLFLowConvertorInterface
 from azureml.model.mgmt.processors.transformers.config import (
     HF_CONF,
@@ -193,29 +196,58 @@ def _save(
             logger.info("Experimental features enabled for MLflow conversion")
             self._hf_conf["exp"] = True
 
-        hf_mlflow.hftransformers.save_model(
-            config=config,
-            tokenizer=tokenizer,
-            hf_model=model,
-            hf_conf=self._hf_conf,
-            conda_env=conda_env,
-            code_paths=code_paths,
-            signature=self._signatures,
-            input_example=input_example,
-            requirements_file=requirements_file,
-            pip_requirements=pip_requirements,
-            extra_pip_requirements=self._extra_pip_requirements,
-            path=self._output_dir,
-        )
+        try:
+            # create a conda environment for OSS transformers Flavor
+            python_version = platform.python_version()
+            pip_pkgs = self._get_curated_environment_pip_package_list()
+            conda_deps = CondaDependencies.create(conda_packages=None,
+                                                  python_version=python_version,
+                                                  pip_packages=pip_pkgs,
+                                                  pin_sdk_version=False)
+            curated_conda_env = conda_deps.as_dict()
+
+            model_pipeline = transformers.pipeline(task=self._task, model=model)
+
+            mlflow.transformers.save_model(
+                transformers_model=model_pipeline,
+                conda_env=curated_conda_env,
+                code_paths=code_paths,
+                signature=self._signatures,
+                input_example=input_example,
+                pip_requirements=pip_requirements,
+                extra_pip_requirements=self._extra_pip_requirements,
+                path=self._output_dir,
+            )
 
-        # move metadata files to parent folder
-        logger.info("Moving meta files such as license, use_policy, readme to parent")
-        move_files(
-            Path(self._output_dir) / "data/model",
-            self._output_dir,
-            include_pattern_str=META_FILE_PATTERN,
-            ignore_case=True
-        )
+            logger.info("Model saved with mlflow OSS flow for task: {}".format(self._task))
+        except Exception as e:
+            logger.error("Model save failed with mlflow OSS flow for task: {} "
+                         "with exception: {}".format(self._task, e))
+
+            hf_mlflow.hftransformers.save_model(
+                config=config,
+                tokenizer=tokenizer,
+                hf_model=model,
+                hf_conf=self._hf_conf,
+                conda_env=conda_env,
+                code_paths=code_paths,
+                signature=self._signatures,
+                input_example=input_example,
+                requirements_file=requirements_file,
+                pip_requirements=pip_requirements,
+                extra_pip_requirements=self._extra_pip_requirements,
+                path=self._output_dir,
+            )
+
+            logger.info("Model saved with transformers evaluate flow for task: {}".format(self._task))
+            # move metadata files to parent folder
+            logger.info("Moving meta files such as license, use_policy, readme to parent")
+            move_files(
+                Path(self._output_dir) / "data/model",
+                self._output_dir,
+                include_pattern_str=META_FILE_PATTERN,
+                ignore_case=True
+            )
 
         # pin pycocotools==2.0.4
         self._update_conda_dependencies({"pycocotools": "2.0.4"})
@@ -253,6 +285,42 @@ def _update_conda_dependencies(self, package_details):
             yaml.safe_dump(conda_dict, f)
             logger.info("updated conda.yaml")
 
+    def _get_curated_environment_pip_package_list(self) -> List[str]:
+        """
+        Retrieve the packages using 'conda list' command.
+
+        :return: A List of the pip package and the corresponding versions.
+        """
+        import subprocess
+        import json
+
+        PIP_LIST = ['mlflow', 'accelerate', 'cffi', 'dill', 'google-api-core', 'numpy',
+                    'packaging', 'pillow', 'protobuf', 'pyyaml', 'requests', 'scikit-learn',
+                    'scipy', 'sentencepiece', 'torch', 'transformers']
+        ADD_PACKAGE_LIST = ['torchvision==0.14.1']
+
+        conda_list_cmd = ["conda", "list", "--json"]
+        try:
+            process = subprocess.run(conda_list_cmd, shell=False, check=True,
+                                     stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        except (FileNotFoundError, subprocess.CalledProcessError) as err:
+            logger.warning('subprocess failed to get dependencies list from conda with error: {}'.format(err))
+            return []
+        output_str = process.stdout.decode('ascii')
+        output_json = json.loads(output_str)
+        pip_list = []
+        for pkg in output_json:
+            pkg_name = pkg['name']
+            pkg_version = pkg['version']
+            if pkg_name in PIP_LIST:
+                pip_list.append(pkg_name + "==" + pkg_version)
+
+        for pkg in ADD_PACKAGE_LIST:
+            pip_list.append(pkg)
+
+        logger.info("pip list: {}".format(pip_list))
+        return pip_list
+
     def _validate(self, translate_params):
         if not translate_params.get("task"):
             raise Exception("task is a required parameter for hftransformers MLflow flavor.")
@@ -412,4 +480,4 @@ def save_as_mlflow(self):
         hf_conf[HF_CONF.HF_PRETRAINED_CLASS.value] = self._hf_model_cls.__name__
         hf_conf[HF_CONF.HF_TOKENIZER_CLASS.value] = self._hf_tokenizer_cls.__name__
 
-        return super()._save(segregate=True)
+        return super()._save(segregate=False)