Merge branch 'main' into ayushmishra/batch-score-oss

Azure · Sep 24, 2024 · dceff97 · dceff97
2 parents 8d858d4 + 5e290e4
commit dceff97
Show file tree

Hide file tree

Showing 46 changed files with 282 additions and 126 deletions.
diff --git a/.github/workflows/assets-validation.yaml b/.github/workflows/assets-validation.yaml
@@ -7,8 +7,6 @@ on:
   pull_request:
     branches:
       - main
-    paths:
-      - assets/**
   workflow_dispatch:
     inputs:
       asset_dirs:
@@ -30,24 +28,27 @@ defaults:
 permissions:
   # Required for OIDC login to Azure
   id-token: write
+  # Required to clone repo
+  contents: read
 
 jobs:
+  check-directory-file-changes:
+    uses: Azure/azureml-assets/.github/workflows/check-changed-files.yaml@main
+    with:
+      folder_path: "assets/**"
+      forked_pr_input: false
+
   validate:
     name: Validate
+    needs: check-directory-file-changes
+    if: needs.check-directory-file-changes.outputs.files-changed != ''
     runs-on: ubuntu-latest
     environment: Testing
 
     steps:
       - name: Clone branch
         uses: actions/checkout@v3
 
-      - name: Get changed files
-        id: changed-files
-        uses: tj-actions/changed-files@v35
-        with:
-          files_separator: ','
-          separator: ','
-
       - name: Use Python 3.10 or newer
         uses: actions/setup-python@v4
         with:
@@ -70,7 +71,7 @@ jobs:
         run: pip install -e $scripts_azureml_assets_dir
 
       - name: Validate assets
-        run: python -u $scripts_assets_dir/validate_assets.py -i "${{ github.event.inputs.asset_dirs || env.default_asset_dirs }}" -a $asset_config_filename -c "${{ steps.changed-files.outputs.all_modified_files }}" -n -I -C -b -t -e
+        run: python -u $scripts_assets_dir/validate_assets.py -i "${{ github.event.inputs.asset_dirs || env.default_asset_dirs }}" -a $asset_config_filename -c "${{ needs.check-directory-file-changes.outputs.files-changed }}" -n -I -C -b -t -e
         env:
           SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
 

diff --git a/assets/common/components/mlflow_model_local_validation/spec.yaml b/assets/common/components/mlflow_model_local_validation/spec.yaml
@@ -1,15 +1,15 @@
 $schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json
 
 name: mlflow_model_local_validation
-version: 0.0.14
+version: 0.0.15
 type: command
 
 is_deterministic: True
 
 display_name: MLFlow model local validation
 description: Validates if a MLFLow model can be loaded on a compute and is usable for inferencing.
 
-environment: azureml://registries/azureml/environments/python-sdk-v2/versions/21
+environment: azureml://registries/azureml/environments/python-sdk-v2/versions/22
 
 code: ../../src/
 command: >-

diff --git a/assets/common/components/register_model/spec.yaml b/assets/common/components/register_model/spec.yaml
@@ -1,6 +1,6 @@
 $schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json
 name: register_model
-version: 0.0.17
+version: 0.0.18
 type: command
 
 is_deterministic: True
@@ -9,7 +9,7 @@ display_name: Register model
 description:
   Register a model to a workspace or a registry. The component works on compute with [MSI](https://learn.microsoft.com/en-us/azure/machine-learning/how-to-create-manage-compute-instance?tabs=python) attached.
 
-environment: azureml://registries/azureml/environments/python-sdk-v2/versions/21
+environment: azureml://registries/azureml/environments/python-sdk-v2/versions/22
 
 code: ../../src
 command: >-

diff --git a/assets/large_language_models/rag/components/crack_and_chunk/spec.yaml b/assets/large_language_models/rag/components/crack_and_chunk/spec.yaml
@@ -4,7 +4,7 @@ type: command
 tags:
     Preview: ""
 
-version: 0.0.71
+version: 0.0.72
 name: llm_rag_crack_and_chunk
 display_name: LLM - Crack and Chunk Data
 is_deterministic: true

diff --git a/assets/large_language_models/rag/components/crack_and_chunk_and_embed/spec.yaml b/assets/large_language_models/rag/components/crack_and_chunk_and_embed/spec.yaml
@@ -4,7 +4,7 @@ type: command
 tags:
     Preview: ""
 
-version: 0.0.42
+version: 0.0.43
 name: llm_rag_crack_and_chunk_and_embed
 display_name: LLM - Crack, Chunk and Embed Data
 is_deterministic: true

diff --git a/assets/large_language_models/rag/components/crack_chunk_embed_index_and_register/spec.yaml b/assets/large_language_models/rag/components/crack_chunk_embed_index_and_register/spec.yaml
@@ -1,5 +1,5 @@
 name: llm_rag_crack_chunk_embed_index_and_register
-version: 0.0.30
+version: 0.0.31
 tags:
     Preview: ""
 

diff --git a/assets/large_language_models/rag/components/crawl_url/spec.yaml b/assets/large_language_models/rag/components/crawl_url/spec.yaml
@@ -4,7 +4,7 @@ type: command
 tags:
     Preview: ""
 
-version: 0.0.29
+version: 0.0.30
 name: llm_rag_crawl_url
 display_name: LLM - Crawl URL to Retrieve Data
 is_deterministic: true

diff --git a/assets/large_language_models/rag/components/create_faiss_index/spec.yaml b/assets/large_language_models/rag/components/create_faiss_index/spec.yaml
@@ -4,7 +4,7 @@ type: command
 tags:
     Preview: ""
 
-version: 0.0.69
+version: 0.0.70
 name: llm_rag_create_faiss_index
 display_name: LLM - Create FAISS Index
 is_deterministic: true

diff --git a/assets/large_language_models/rag/components/create_promptflow/spec.yaml b/assets/large_language_models/rag/components/create_promptflow/spec.yaml
@@ -4,7 +4,7 @@ type: command
 tags:
     Preview: ""
 
-version: 0.0.80
+version: 0.0.81
 name: llm_rag_create_promptflow
 display_name: LLM - Create Prompt Flow
 is_deterministic: true

diff --git a/assets/large_language_models/rag/components/data_import_acs/spec.yaml b/assets/large_language_models/rag/components/data_import_acs/spec.yaml
@@ -4,7 +4,7 @@ type: command
 tags:
     Preview: ""
 
-version: 0.0.65
+version: 0.0.66
 name: llm_rag_data_import_acs
 display_name: LLM - Import Data from ACS
 is_deterministic: false

diff --git a/assets/large_language_models/rag/components/generate_embeddings/spec.yaml b/assets/large_language_models/rag/components/generate_embeddings/spec.yaml
@@ -4,7 +4,7 @@ type: command
 tags:
     Preview: ""
 
-version: 0.0.64
+version: 0.0.65
 name: llm_rag_generate_embeddings
 display_name: LLM - Generate Embeddings
 is_deterministic: true

diff --git a/assets/large_language_models/rag/components/generate_embeddings_parallel/spec.yaml b/assets/large_language_models/rag/components/generate_embeddings_parallel/spec.yaml
@@ -4,7 +4,7 @@ type: parallel
 tags:
     Preview: ""
 
-version: 0.0.70
+version: 0.0.71
 name: llm_rag_generate_embeddings_parallel
 display_name: LLM - Generate Embeddings Parallel
 is_deterministic: true

diff --git a/assets/large_language_models/rag/components/git_clone/spec.yaml b/assets/large_language_models/rag/components/git_clone/spec.yaml
@@ -4,7 +4,7 @@ type: command
 tags:
     Preview: ""
 
-version: 0.0.68
+version: 0.0.69
 name: llm_rag_git_clone
 display_name: LLM - Clone Git Repo
 is_deterministic: true

diff --git a/assets/large_language_models/rag/components/image_embed_index/spec.yaml b/assets/large_language_models/rag/components/image_embed_index/spec.yaml
@@ -4,7 +4,7 @@ type: command
 tags:
     Preview: ""
 
-version: 0.0.22
+version: 0.0.23
 name: llm_rag_image_embed_index
 display_name: LLM - Embedding images with Florence 
 is_deterministic: true

diff --git a/assets/large_language_models/rag/components/qa_data_generation/spec.yaml b/assets/large_language_models/rag/components/qa_data_generation/spec.yaml
@@ -4,7 +4,7 @@ type: command
 tags:
     Preview: ""
 
-version: 0.0.68
+version: 0.0.69
 name: llm_rag_qa_data_generation
 display_name: LLM - Generate QnA Test Data
 is_deterministic: true

diff --git a/assets/large_language_models/rag/components/register_mlindex_asset/spec.yaml b/assets/large_language_models/rag/components/register_mlindex_asset/spec.yaml
@@ -4,7 +4,7 @@ type: command
 tags:
     Preview: ""
 
-version: 0.0.68
+version: 0.0.69
 name: llm_rag_register_mlindex_asset
 display_name: LLM - Register Vector Index Asset
 is_deterministic: true

diff --git a/assets/large_language_models/rag/components/register_qa_data_asset/spec.yaml b/assets/large_language_models/rag/components/register_qa_data_asset/spec.yaml
@@ -4,7 +4,7 @@ type: command
 tags:
     Preview: ""
 
-version: 0.0.61
+version: 0.0.62
 name: llm_rag_register_qa_data_asset
 display_name: LLM - Register QA Generation Data Asset
 is_deterministic: true

diff --git a/assets/large_language_models/rag/components/update_acs_index/spec.yaml b/assets/large_language_models/rag/components/update_acs_index/spec.yaml
@@ -4,7 +4,7 @@ type: command
 tags:
     Preview: ""
 
-version: 0.0.68
+version: 0.0.69
 name: llm_rag_update_acs_index
 display_name: LLM - Update ACS Index
 is_deterministic: true

diff --git a/assets/large_language_models/rag/components/update_azure_cosmos_mongo_vcore_index/spec.yaml b/assets/large_language_models/rag/components/update_azure_cosmos_mongo_vcore_index/spec.yaml
@@ -4,7 +4,7 @@ type: command
 tags:
     Preview: ""
 
-version: 0.0.24
+version: 0.0.25
 name: llm_rag_update_cosmos_mongo_vcore_index
 display_name: LLM - Update Azure Cosmos Mongo vCore Index
 is_deterministic: true

diff --git a/assets/large_language_models/rag/components/update_milvus_index/spec.yaml b/assets/large_language_models/rag/components/update_milvus_index/spec.yaml
@@ -4,7 +4,7 @@ type: command
 tags:
     Preview: ""
 
-version: 0.0.24
+version: 0.0.25
 name: llm_rag_update_milvus_index
 display_name: LLM - Update Milvus Index
 is_deterministic: true

diff --git a/assets/large_language_models/rag/components/update_pinecone_index/spec.yaml b/assets/large_language_models/rag/components/update_pinecone_index/spec.yaml
@@ -4,7 +4,7 @@ type: command
 tags:
     Preview: ""
 
-version: 0.0.36
+version: 0.0.37
 name: llm_rag_update_pinecone_index
 display_name: LLM - Update Pinecone Index
 is_deterministic: true

diff --git a/assets/large_language_models/rag/components/validate_deployments/spec.yaml b/assets/large_language_models/rag/components/validate_deployments/spec.yaml
@@ -4,7 +4,7 @@ type: command
 tags:
     Preview: ""
 
-version: 0.0.74
+version: 0.0.75
 name: llm_rag_validate_deployments
 display_name: LLM - Validate Deployments
 is_deterministic: false

diff --git a/assets/large_language_models/rag/environments/rag/context/conda_dependencies.yaml b/assets/large_language_models/rag/environments/rag/context/conda_dependencies.yaml
@@ -25,7 +25,7 @@ dependencies:
   - azureml-mlflow=={{latest-pypi-version}}
     # Other public packages
   - beautifulsoup4~=4.11.2
-  - cryptography>=41.0.7
+  - cryptography>=43.0.1
   - datasets~=2.10.1
   - faiss-cpu~=1.7.3
   - inference-schema>=1.8.0
@@ -34,8 +34,8 @@ dependencies:
   - markdown
   - mlflow-skinny==2.3.2
   - mlflow>=2.6.0
-  - msal~=1.24.0
-  - nltk==3.8.1
+  - msal~=1.31.0
+  - nltk==3.9.1
   - openai>=0.27.4
   - pandas>=1
   - polling2~=0.5.0

diff --git a/assets/large_language_models/rag/environments/rag_embeddings/context/conda_dependencies.yaml b/assets/large_language_models/rag/environments/rag_embeddings/context/conda_dependencies.yaml
@@ -35,9 +35,9 @@ dependencies:
   - markdown
   - mlflow>=2.6.0
   - mlflow-skinny==2.3.2
-  - msal~=1.24.0
+  - msal~=1.31.0
   - msrest>=0.6.18
-  - nltk==3.8.1
+  - nltk==3.9.1
   - openai>=0.27.4
   - pandas>=1
   - polling2~=0.5.0
@@ -53,4 +53,4 @@ dependencies:
   - python-dateutil>=2.7.3,<3.0.0
   - PyJWT<3.0.0
   - pytz
-  - cryptography>=41.0.7
+  - cryptography>=43.0.1
diff --git a/assets/promptflow/models/rai-eval-ui-dag-flow/model.yaml b/assets/promptflow/models/rai-eval-ui-dag-flow/model.yaml
@@ -1,6 +1,6 @@
 path:
   container_name: rai-eval-flows
-  container_path: models/rai_eval_ui_dag_flow/v1
+  container_path: models/rai_eval_ui_dag_flow/v2
   storage_name: amlraipfmodels
   type: azureblob
 publish:

diff --git a/assets/promptflow/models/rai-eval-ui-dag-flow/spec.yaml b/assets/promptflow/models/rai-eval-ui-dag-flow/spec.yaml
@@ -9,4 +9,4 @@ properties:
   azureml.promptflow.description: Compute the quality and safety of the answer for the given question based on the ground_truth and the context
   inference-min-sku-spec: 2|0|14|28
   inference-recommended-sku: Standard_DS3_v2
-version: 1
+version: 2
diff --git a/assets/sample/general/components/sample_component/asset.yaml b/assets/sample/general/components/sample_component/asset.yaml
@@ -1,2 +1,3 @@
 type: component
 spec: spec.yaml
+categories: ["Sample"]
diff --git a/assets/sample/general/components/sample_component/spec.yaml b/assets/sample/general/components/sample_component/spec.yaml
@@ -1,7 +1,7 @@
 $schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json
 type: command
 
-name: sample_component
+name: my_sample
 display_name: Hello_Python_World
 version: 0.0.1
 

diff --git a/assets/sample/general/jobs/pipeline.yml b/assets/sample/general/jobs/pipeline.yml
@@ -6,5 +6,5 @@ display_name: Hello-World-Test-Pipeline
 jobs:
   hello_python_world_job:
     type: command
-    component: file:../components/sample_component/spec.yaml
+    component: file:../components/my_sample/spec.yaml
     compute: azureml:cpu-cluster
diff --git a/assets/sample/tests.yml b/assets/sample/tests.yml
@@ -7,4 +7,4 @@ general:
     sample_pytest: 
       pytest_job: "./general/jobs/sample_pytest.py"
       assets:
-      - ./general/components/sample_component
+      - ./general/components/my_sample
diff --git a/assets/training/general/environments/tensorflow-2.16-cuda11/context/Dockerfile b/assets/training/general/environments/tensorflow-2.16-cuda11/context/Dockerfile
@@ -6,6 +6,10 @@ ENV CONDA_PREFIX=/azureml-envs/tensorflow-2.16-cuda11
 ENV CONDA_DEFAULT_ENV=$CONDA_PREFIX
 ENV PATH=$CONDA_PREFIX/bin:$PATH
 
+# Enable debug
+RUN apt-get update
+RUN apt-get install -y openssh-server openssh-client
+
 # Create conda environment
 COPY conda_dependencies.yaml .
 RUN conda env create -p $CONDA_PREFIX -f conda_dependencies.yaml -q && \

diff --git a/assets/training/model_management/components/convert_model_to_mlflow/spec.yaml b/assets/training/model_management/components/convert_model_to_mlflow/spec.yaml
@@ -1,15 +1,15 @@
 $schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json
 
 name: convert_model_to_mlflow
-version: 0.0.31
+version: 0.0.32
 type: command
 
 is_deterministic: True
 
 display_name: Convert models to MLflow
 description: Component converts models from supported frameworks to MLflow model packaging format
 
-environment: azureml://registries/azureml/environments/model-management/versions/32
+environment: azureml://registries/azureml/environments/model-management/versions/34
 
 code: ../../src/
 command: |

diff --git a/assets/training/model_management/components/download_model/spec.yaml b/assets/training/model_management/components/download_model/spec.yaml
@@ -1,15 +1,15 @@
 $schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json
 
 name: download_model
-version: 0.0.28
+version: 0.0.29
 type: command
 
 is_deterministic: True
 
 display_name: Download model
 description: Downloads a publicly available model
 
-environment: azureml://registries/azureml/environments/model-management/versions/32
+environment: azureml://registries/azureml/environments/model-management/versions/34
 
 code: ../../src/
 command: >