From 1889c037b341ca28533a9fb83ccc81f4797b8830 Mon Sep 17 00:00:00 2001 From: QianqianNie <44004519+QianqianNie@users.noreply.github.com> Date: Mon, 23 Sep 2024 09:27:44 -0700 Subject: [PATCH 1/8] Update fmi (#3417) --- .../foundation-model-inference/context/Dockerfile | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/assets/training/model_management/environments/foundation-model-inference/context/Dockerfile b/assets/training/model_management/environments/foundation-model-inference/context/Dockerfile index 98473187ca..e1e2bfc47c 100644 --- a/assets/training/model_management/environments/foundation-model-inference/context/Dockerfile +++ b/assets/training/model_management/environments/foundation-model-inference/context/Dockerfile @@ -47,7 +47,12 @@ RUN pip install git+https://github.com/stanford-futuredata/megablocks.git@5897cd # RUN pip install -e ./ --no-cache-dir # When copied to assets repo, change to install from public pypi -RUN pip install llm-optimized-inference==0.2.11 --no-cache-dir +RUN pip install llm-optimized-inference==0.2.12 --no-cache-dir + +RUN pip uninstall transformers -y +RUN pip uninstall -y vllm +RUN pip install https://automlsamplenotebookdata.blob.core.windows.net/vllm/transformers-4.45.0.dev0-py3-none-any.whl +RUN pip install https://automlsamplenotebookdata.blob.core.windows.net/vllm/vllm-0.6.1.post2+cu124-cp38-abi3-manylinux1_x86_64.whl # clean conda and pip caches RUN rm -rf ~/.cache/pip @@ -59,4 +64,4 @@ RUN chmod +x /var/runit/api_server/run ENV SVDIR=/var/runit ENV WORKER_TIMEOUT=3600 EXPOSE 5001 -CMD [ "runsvdir", "/var/runit" ] +CMD [ "runsvdir", "/var/runit" ] \ No newline at end of file From dd5058c04833173a533d715aaeea090d016ce226 Mon Sep 17 00:00:00 2001 From: Kelly <40868256+lykelly19@users.noreply.github.com> Date: Mon, 23 Sep 2024 12:31:04 -0400 Subject: [PATCH 2/8] Update assets validation to use check-changed-files job to be able to require status check (#3333) * Update assets validation to run on all assets * Rename from sample_component to my_sample * update sample component * use new check-directory-file-changes job * update read access * update forked_pr_input * add missing quote --------- Co-authored-by: lykelly19 --- .github/workflows/assets-validation.yaml | 21 ++++++++++--------- .../components/sample_component/asset.yaml | 1 + .../components/sample_component/spec.yaml | 2 +- assets/sample/general/jobs/pipeline.yml | 2 +- assets/sample/tests.yml | 2 +- 5 files changed, 15 insertions(+), 13 deletions(-) diff --git a/.github/workflows/assets-validation.yaml b/.github/workflows/assets-validation.yaml index 5e63e274d4..6f8c5caaf3 100644 --- a/.github/workflows/assets-validation.yaml +++ b/.github/workflows/assets-validation.yaml @@ -7,8 +7,6 @@ on: pull_request: branches: - main - paths: - - assets/** workflow_dispatch: inputs: asset_dirs: @@ -30,10 +28,20 @@ defaults: permissions: # Required for OIDC login to Azure id-token: write + # Required to clone repo + contents: read jobs: + check-directory-file-changes: + uses: Azure/azureml-assets/.github/workflows/check-changed-files.yaml@main + with: + folder_path: "assets/**" + forked_pr_input: false + validate: name: Validate + needs: check-directory-file-changes + if: needs.check-directory-file-changes.outputs.files-changed != '' runs-on: ubuntu-latest environment: Testing @@ -41,13 +49,6 @@ jobs: - name: Clone branch uses: actions/checkout@v3 - - name: Get changed files - id: changed-files - uses: tj-actions/changed-files@v35 - with: - files_separator: ',' - separator: ',' - - name: Use Python 3.10 or newer uses: actions/setup-python@v4 with: @@ -70,7 +71,7 @@ jobs: run: pip install -e $scripts_azureml_assets_dir - name: Validate assets - run: python -u $scripts_assets_dir/validate_assets.py -i "${{ github.event.inputs.asset_dirs || env.default_asset_dirs }}" -a $asset_config_filename -c "${{ steps.changed-files.outputs.all_modified_files }}" -n -I -C -b -t -e + run: python -u $scripts_assets_dir/validate_assets.py -i "${{ github.event.inputs.asset_dirs || env.default_asset_dirs }}" -a $asset_config_filename -c "${{ needs.check-directory-file-changes.outputs.files-changed }}" -n -I -C -b -t -e env: SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} diff --git a/assets/sample/general/components/sample_component/asset.yaml b/assets/sample/general/components/sample_component/asset.yaml index d1698f05cc..d5ec8e5e67 100644 --- a/assets/sample/general/components/sample_component/asset.yaml +++ b/assets/sample/general/components/sample_component/asset.yaml @@ -1,2 +1,3 @@ type: component spec: spec.yaml +categories: ["Sample"] \ No newline at end of file diff --git a/assets/sample/general/components/sample_component/spec.yaml b/assets/sample/general/components/sample_component/spec.yaml index 38ecc16316..c466801d0e 100644 --- a/assets/sample/general/components/sample_component/spec.yaml +++ b/assets/sample/general/components/sample_component/spec.yaml @@ -1,7 +1,7 @@ $schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json type: command -name: sample_component +name: my_sample display_name: Hello_Python_World version: 0.0.1 diff --git a/assets/sample/general/jobs/pipeline.yml b/assets/sample/general/jobs/pipeline.yml index 60c6134863..c0daa8957a 100644 --- a/assets/sample/general/jobs/pipeline.yml +++ b/assets/sample/general/jobs/pipeline.yml @@ -6,5 +6,5 @@ display_name: Hello-World-Test-Pipeline jobs: hello_python_world_job: type: command - component: file:../components/sample_component/spec.yaml + component: file:../components/my_sample/spec.yaml compute: azureml:cpu-cluster diff --git a/assets/sample/tests.yml b/assets/sample/tests.yml index db968245ac..495b9dbb4e 100644 --- a/assets/sample/tests.yml +++ b/assets/sample/tests.yml @@ -7,4 +7,4 @@ general: sample_pytest: pytest_job: "./general/jobs/sample_pytest.py" assets: - - ./general/components/sample_component + - ./general/components/my_sample From 440a67971b6f66af3e45a9d6b10598ce531a52d1 Mon Sep 17 00:00:00 2001 From: jeff-shepherd <39775772+jeff-shepherd@users.noreply.github.com> Date: Mon, 23 Sep 2024 10:15:01 -0700 Subject: [PATCH 3/8] Enable debug for tensorflow image (#3419) --- .../environments/tensorflow-2.16-cuda11/context/Dockerfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/assets/training/general/environments/tensorflow-2.16-cuda11/context/Dockerfile b/assets/training/general/environments/tensorflow-2.16-cuda11/context/Dockerfile index 401238232f..84b7dd7503 100644 --- a/assets/training/general/environments/tensorflow-2.16-cuda11/context/Dockerfile +++ b/assets/training/general/environments/tensorflow-2.16-cuda11/context/Dockerfile @@ -6,6 +6,10 @@ ENV CONDA_PREFIX=/azureml-envs/tensorflow-2.16-cuda11 ENV CONDA_DEFAULT_ENV=$CONDA_PREFIX ENV PATH=$CONDA_PREFIX/bin:$PATH +# Enable debug +RUN apt-get update +RUN apt-get install -y openssh-server openssh-client + # Create conda environment COPY conda_dependencies.yaml . RUN conda env create -p $CONDA_PREFIX -f conda_dependencies.yaml -q && \ From 1984620b697bdbe94580cecfe6c3a22c0e93d1fe Mon Sep 17 00:00:00 2001 From: tongy-msft <91754176+tongyu-microsoft@users.noreply.github.com> Date: Mon, 23 Sep 2024 16:13:30 -0700 Subject: [PATCH 4/8] Release rai_eval_ui_dag_flow v2 to make connection optional for generate answer node (#3423) * Update model.yaml * Update spec.yaml --- assets/promptflow/models/rai-eval-ui-dag-flow/model.yaml | 2 +- assets/promptflow/models/rai-eval-ui-dag-flow/spec.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/assets/promptflow/models/rai-eval-ui-dag-flow/model.yaml b/assets/promptflow/models/rai-eval-ui-dag-flow/model.yaml index 0047621eff..dd58915c02 100644 --- a/assets/promptflow/models/rai-eval-ui-dag-flow/model.yaml +++ b/assets/promptflow/models/rai-eval-ui-dag-flow/model.yaml @@ -1,6 +1,6 @@ path: container_name: rai-eval-flows - container_path: models/rai_eval_ui_dag_flow/v1 + container_path: models/rai_eval_ui_dag_flow/v2 storage_name: amlraipfmodels type: azureblob publish: diff --git a/assets/promptflow/models/rai-eval-ui-dag-flow/spec.yaml b/assets/promptflow/models/rai-eval-ui-dag-flow/spec.yaml index ff1f15ff9d..1b2858a8c3 100644 --- a/assets/promptflow/models/rai-eval-ui-dag-flow/spec.yaml +++ b/assets/promptflow/models/rai-eval-ui-dag-flow/spec.yaml @@ -9,4 +9,4 @@ properties: azureml.promptflow.description: Compute the quality and safety of the answer for the given question based on the ground_truth and the context inference-min-sku-spec: 2|0|14|28 inference-recommended-sku: Standard_DS3_v2 -version: 1 +version: 2 From 3e7589ef701b3ac53b4bc3002718fe8567f42ba3 Mon Sep 17 00:00:00 2001 From: runhli <87873226+runhli@users.noreply.github.com> Date: Mon, 23 Sep 2024 20:09:31 -0700 Subject: [PATCH 5/8] upgrade rag component env to fix vulnerable images icm (#3422) * upgrade rag env to fix vulnerable images icm * try * upgrade * upgrade ntkl --- .../rag/components/crack_and_chunk/spec.yaml | 2 +- .../rag/components/crack_and_chunk_and_embed/spec.yaml | 2 +- .../crack_chunk_embed_index_and_register/spec.yaml | 2 +- .../rag/components/crawl_url/spec.yaml | 2 +- .../rag/components/create_faiss_index/spec.yaml | 2 +- .../rag/components/create_promptflow/spec.yaml | 2 +- .../rag/components/data_import_acs/spec.yaml | 2 +- .../rag/components/generate_embeddings/spec.yaml | 2 +- .../rag/components/generate_embeddings_parallel/spec.yaml | 2 +- .../rag/components/git_clone/spec.yaml | 2 +- .../rag/components/image_embed_index/spec.yaml | 2 +- .../rag/components/qa_data_generation/spec.yaml | 2 +- .../rag/components/register_mlindex_asset/spec.yaml | 2 +- .../rag/components/register_qa_data_asset/spec.yaml | 2 +- .../rag/components/update_acs_index/spec.yaml | 2 +- .../update_azure_cosmos_mongo_vcore_index/spec.yaml | 2 +- .../rag/components/update_milvus_index/spec.yaml | 2 +- .../rag/components/update_pinecone_index/spec.yaml | 2 +- .../rag/components/validate_deployments/spec.yaml | 2 +- .../rag/environments/rag/context/conda_dependencies.yaml | 6 +++--- .../rag_embeddings/context/conda_dependencies.yaml | 6 +++--- 21 files changed, 25 insertions(+), 25 deletions(-) diff --git a/assets/large_language_models/rag/components/crack_and_chunk/spec.yaml b/assets/large_language_models/rag/components/crack_and_chunk/spec.yaml index 0ec5514c4a..6a09c82ce6 100644 --- a/assets/large_language_models/rag/components/crack_and_chunk/spec.yaml +++ b/assets/large_language_models/rag/components/crack_and_chunk/spec.yaml @@ -4,7 +4,7 @@ type: command tags: Preview: "" -version: 0.0.71 +version: 0.0.72 name: llm_rag_crack_and_chunk display_name: LLM - Crack and Chunk Data is_deterministic: true diff --git a/assets/large_language_models/rag/components/crack_and_chunk_and_embed/spec.yaml b/assets/large_language_models/rag/components/crack_and_chunk_and_embed/spec.yaml index 62bf3432f2..aad846a041 100644 --- a/assets/large_language_models/rag/components/crack_and_chunk_and_embed/spec.yaml +++ b/assets/large_language_models/rag/components/crack_and_chunk_and_embed/spec.yaml @@ -4,7 +4,7 @@ type: command tags: Preview: "" -version: 0.0.42 +version: 0.0.43 name: llm_rag_crack_and_chunk_and_embed display_name: LLM - Crack, Chunk and Embed Data is_deterministic: true diff --git a/assets/large_language_models/rag/components/crack_chunk_embed_index_and_register/spec.yaml b/assets/large_language_models/rag/components/crack_chunk_embed_index_and_register/spec.yaml index 2603086bd4..81a05fd3de 100644 --- a/assets/large_language_models/rag/components/crack_chunk_embed_index_and_register/spec.yaml +++ b/assets/large_language_models/rag/components/crack_chunk_embed_index_and_register/spec.yaml @@ -1,5 +1,5 @@ name: llm_rag_crack_chunk_embed_index_and_register -version: 0.0.30 +version: 0.0.31 tags: Preview: "" diff --git a/assets/large_language_models/rag/components/crawl_url/spec.yaml b/assets/large_language_models/rag/components/crawl_url/spec.yaml index b4f2712cbc..c2740cb7ef 100644 --- a/assets/large_language_models/rag/components/crawl_url/spec.yaml +++ b/assets/large_language_models/rag/components/crawl_url/spec.yaml @@ -4,7 +4,7 @@ type: command tags: Preview: "" -version: 0.0.29 +version: 0.0.30 name: llm_rag_crawl_url display_name: LLM - Crawl URL to Retrieve Data is_deterministic: true diff --git a/assets/large_language_models/rag/components/create_faiss_index/spec.yaml b/assets/large_language_models/rag/components/create_faiss_index/spec.yaml index 7ca62a65a4..2118282a73 100644 --- a/assets/large_language_models/rag/components/create_faiss_index/spec.yaml +++ b/assets/large_language_models/rag/components/create_faiss_index/spec.yaml @@ -4,7 +4,7 @@ type: command tags: Preview: "" -version: 0.0.69 +version: 0.0.70 name: llm_rag_create_faiss_index display_name: LLM - Create FAISS Index is_deterministic: true diff --git a/assets/large_language_models/rag/components/create_promptflow/spec.yaml b/assets/large_language_models/rag/components/create_promptflow/spec.yaml index 5dd20e1ce5..64d61be4b0 100644 --- a/assets/large_language_models/rag/components/create_promptflow/spec.yaml +++ b/assets/large_language_models/rag/components/create_promptflow/spec.yaml @@ -4,7 +4,7 @@ type: command tags: Preview: "" -version: 0.0.80 +version: 0.0.81 name: llm_rag_create_promptflow display_name: LLM - Create Prompt Flow is_deterministic: true diff --git a/assets/large_language_models/rag/components/data_import_acs/spec.yaml b/assets/large_language_models/rag/components/data_import_acs/spec.yaml index 1292cfe89f..db4c36319e 100644 --- a/assets/large_language_models/rag/components/data_import_acs/spec.yaml +++ b/assets/large_language_models/rag/components/data_import_acs/spec.yaml @@ -4,7 +4,7 @@ type: command tags: Preview: "" -version: 0.0.65 +version: 0.0.66 name: llm_rag_data_import_acs display_name: LLM - Import Data from ACS is_deterministic: false diff --git a/assets/large_language_models/rag/components/generate_embeddings/spec.yaml b/assets/large_language_models/rag/components/generate_embeddings/spec.yaml index b58c3c5c35..c784e901de 100644 --- a/assets/large_language_models/rag/components/generate_embeddings/spec.yaml +++ b/assets/large_language_models/rag/components/generate_embeddings/spec.yaml @@ -4,7 +4,7 @@ type: command tags: Preview: "" -version: 0.0.64 +version: 0.0.65 name: llm_rag_generate_embeddings display_name: LLM - Generate Embeddings is_deterministic: true diff --git a/assets/large_language_models/rag/components/generate_embeddings_parallel/spec.yaml b/assets/large_language_models/rag/components/generate_embeddings_parallel/spec.yaml index a6719a802c..33ed224ed1 100644 --- a/assets/large_language_models/rag/components/generate_embeddings_parallel/spec.yaml +++ b/assets/large_language_models/rag/components/generate_embeddings_parallel/spec.yaml @@ -4,7 +4,7 @@ type: parallel tags: Preview: "" -version: 0.0.70 +version: 0.0.71 name: llm_rag_generate_embeddings_parallel display_name: LLM - Generate Embeddings Parallel is_deterministic: true diff --git a/assets/large_language_models/rag/components/git_clone/spec.yaml b/assets/large_language_models/rag/components/git_clone/spec.yaml index 3a9c958ab7..307a71835a 100644 --- a/assets/large_language_models/rag/components/git_clone/spec.yaml +++ b/assets/large_language_models/rag/components/git_clone/spec.yaml @@ -4,7 +4,7 @@ type: command tags: Preview: "" -version: 0.0.68 +version: 0.0.69 name: llm_rag_git_clone display_name: LLM - Clone Git Repo is_deterministic: true diff --git a/assets/large_language_models/rag/components/image_embed_index/spec.yaml b/assets/large_language_models/rag/components/image_embed_index/spec.yaml index 33cb5c8fd3..e29ad5ad38 100644 --- a/assets/large_language_models/rag/components/image_embed_index/spec.yaml +++ b/assets/large_language_models/rag/components/image_embed_index/spec.yaml @@ -4,7 +4,7 @@ type: command tags: Preview: "" -version: 0.0.22 +version: 0.0.23 name: llm_rag_image_embed_index display_name: LLM - Embedding images with Florence is_deterministic: true diff --git a/assets/large_language_models/rag/components/qa_data_generation/spec.yaml b/assets/large_language_models/rag/components/qa_data_generation/spec.yaml index f228ab517f..9bcfa45b38 100644 --- a/assets/large_language_models/rag/components/qa_data_generation/spec.yaml +++ b/assets/large_language_models/rag/components/qa_data_generation/spec.yaml @@ -4,7 +4,7 @@ type: command tags: Preview: "" -version: 0.0.68 +version: 0.0.69 name: llm_rag_qa_data_generation display_name: LLM - Generate QnA Test Data is_deterministic: true diff --git a/assets/large_language_models/rag/components/register_mlindex_asset/spec.yaml b/assets/large_language_models/rag/components/register_mlindex_asset/spec.yaml index 00795ceb45..f65160f741 100644 --- a/assets/large_language_models/rag/components/register_mlindex_asset/spec.yaml +++ b/assets/large_language_models/rag/components/register_mlindex_asset/spec.yaml @@ -4,7 +4,7 @@ type: command tags: Preview: "" -version: 0.0.68 +version: 0.0.69 name: llm_rag_register_mlindex_asset display_name: LLM - Register Vector Index Asset is_deterministic: true diff --git a/assets/large_language_models/rag/components/register_qa_data_asset/spec.yaml b/assets/large_language_models/rag/components/register_qa_data_asset/spec.yaml index 9363a0660f..22fb34aff5 100644 --- a/assets/large_language_models/rag/components/register_qa_data_asset/spec.yaml +++ b/assets/large_language_models/rag/components/register_qa_data_asset/spec.yaml @@ -4,7 +4,7 @@ type: command tags: Preview: "" -version: 0.0.61 +version: 0.0.62 name: llm_rag_register_qa_data_asset display_name: LLM - Register QA Generation Data Asset is_deterministic: true diff --git a/assets/large_language_models/rag/components/update_acs_index/spec.yaml b/assets/large_language_models/rag/components/update_acs_index/spec.yaml index 382c3c27f3..e1d0c0d54a 100644 --- a/assets/large_language_models/rag/components/update_acs_index/spec.yaml +++ b/assets/large_language_models/rag/components/update_acs_index/spec.yaml @@ -4,7 +4,7 @@ type: command tags: Preview: "" -version: 0.0.68 +version: 0.0.69 name: llm_rag_update_acs_index display_name: LLM - Update ACS Index is_deterministic: true diff --git a/assets/large_language_models/rag/components/update_azure_cosmos_mongo_vcore_index/spec.yaml b/assets/large_language_models/rag/components/update_azure_cosmos_mongo_vcore_index/spec.yaml index 0d8be91845..f88844376a 100644 --- a/assets/large_language_models/rag/components/update_azure_cosmos_mongo_vcore_index/spec.yaml +++ b/assets/large_language_models/rag/components/update_azure_cosmos_mongo_vcore_index/spec.yaml @@ -4,7 +4,7 @@ type: command tags: Preview: "" -version: 0.0.24 +version: 0.0.25 name: llm_rag_update_cosmos_mongo_vcore_index display_name: LLM - Update Azure Cosmos Mongo vCore Index is_deterministic: true diff --git a/assets/large_language_models/rag/components/update_milvus_index/spec.yaml b/assets/large_language_models/rag/components/update_milvus_index/spec.yaml index b66b0bb157..8dee57409a 100644 --- a/assets/large_language_models/rag/components/update_milvus_index/spec.yaml +++ b/assets/large_language_models/rag/components/update_milvus_index/spec.yaml @@ -4,7 +4,7 @@ type: command tags: Preview: "" -version: 0.0.24 +version: 0.0.25 name: llm_rag_update_milvus_index display_name: LLM - Update Milvus Index is_deterministic: true diff --git a/assets/large_language_models/rag/components/update_pinecone_index/spec.yaml b/assets/large_language_models/rag/components/update_pinecone_index/spec.yaml index a64290db56..7cd200164b 100644 --- a/assets/large_language_models/rag/components/update_pinecone_index/spec.yaml +++ b/assets/large_language_models/rag/components/update_pinecone_index/spec.yaml @@ -4,7 +4,7 @@ type: command tags: Preview: "" -version: 0.0.36 +version: 0.0.37 name: llm_rag_update_pinecone_index display_name: LLM - Update Pinecone Index is_deterministic: true diff --git a/assets/large_language_models/rag/components/validate_deployments/spec.yaml b/assets/large_language_models/rag/components/validate_deployments/spec.yaml index 3f469cfc6a..85106424f9 100644 --- a/assets/large_language_models/rag/components/validate_deployments/spec.yaml +++ b/assets/large_language_models/rag/components/validate_deployments/spec.yaml @@ -4,7 +4,7 @@ type: command tags: Preview: "" -version: 0.0.74 +version: 0.0.75 name: llm_rag_validate_deployments display_name: LLM - Validate Deployments is_deterministic: false diff --git a/assets/large_language_models/rag/environments/rag/context/conda_dependencies.yaml b/assets/large_language_models/rag/environments/rag/context/conda_dependencies.yaml index 5b323aea03..45f5d8380c 100644 --- a/assets/large_language_models/rag/environments/rag/context/conda_dependencies.yaml +++ b/assets/large_language_models/rag/environments/rag/context/conda_dependencies.yaml @@ -25,7 +25,7 @@ dependencies: - azureml-mlflow=={{latest-pypi-version}} # Other public packages - beautifulsoup4~=4.11.2 - - cryptography>=41.0.7 + - cryptography>=43.0.1 - datasets~=2.10.1 - faiss-cpu~=1.7.3 - inference-schema>=1.8.0 @@ -34,8 +34,8 @@ dependencies: - markdown - mlflow-skinny==2.3.2 - mlflow>=2.6.0 - - msal~=1.24.0 - - nltk==3.8.1 + - msal~=1.31.0 + - nltk==3.9.1 - openai>=0.27.4 - pandas>=1 - polling2~=0.5.0 diff --git a/assets/large_language_models/rag/environments/rag_embeddings/context/conda_dependencies.yaml b/assets/large_language_models/rag/environments/rag_embeddings/context/conda_dependencies.yaml index 8c603a65e3..b419015730 100644 --- a/assets/large_language_models/rag/environments/rag_embeddings/context/conda_dependencies.yaml +++ b/assets/large_language_models/rag/environments/rag_embeddings/context/conda_dependencies.yaml @@ -35,9 +35,9 @@ dependencies: - markdown - mlflow>=2.6.0 - mlflow-skinny==2.3.2 - - msal~=1.24.0 + - msal~=1.31.0 - msrest>=0.6.18 - - nltk==3.8.1 + - nltk==3.9.1 - openai>=0.27.4 - pandas>=1 - polling2~=0.5.0 @@ -53,4 +53,4 @@ dependencies: - python-dateutil>=2.7.3,<3.0.0 - PyJWT<3.0.0 - pytz - - cryptography>=41.0.7 + - cryptography>=43.0.1 From 504f207cfd8218d28c68b2cfd6ba550d9a0a0bdf Mon Sep 17 00:00:00 2001 From: abhishekMS2024 Date: Tue, 24 Sep 2024 19:59:25 +0530 Subject: [PATCH 6/8] Update import model component to make use of updated env (#3418) --- .../mlflow_model_local_validation/spec.yaml | 4 ++-- assets/common/components/register_model/spec.yaml | 4 ++-- .../components/convert_model_to_mlflow/spec.yaml | 4 ++-- .../components/download_model/spec.yaml | 4 ++-- .../components/import_model/spec.yaml | 12 ++++++------ .../components/validation_trigger_import/spec.yaml | 4 ++-- 6 files changed, 16 insertions(+), 16 deletions(-) diff --git a/assets/common/components/mlflow_model_local_validation/spec.yaml b/assets/common/components/mlflow_model_local_validation/spec.yaml index fdd50ba033..971187a7f8 100644 --- a/assets/common/components/mlflow_model_local_validation/spec.yaml +++ b/assets/common/components/mlflow_model_local_validation/spec.yaml @@ -1,7 +1,7 @@ $schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json name: mlflow_model_local_validation -version: 0.0.14 +version: 0.0.15 type: command is_deterministic: True @@ -9,7 +9,7 @@ is_deterministic: True display_name: MLFlow model local validation description: Validates if a MLFLow model can be loaded on a compute and is usable for inferencing. -environment: azureml://registries/azureml/environments/python-sdk-v2/versions/21 +environment: azureml://registries/azureml/environments/python-sdk-v2/versions/22 code: ../../src/ command: >- diff --git a/assets/common/components/register_model/spec.yaml b/assets/common/components/register_model/spec.yaml index 35812c1027..f8a08e1301 100644 --- a/assets/common/components/register_model/spec.yaml +++ b/assets/common/components/register_model/spec.yaml @@ -1,6 +1,6 @@ $schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json name: register_model -version: 0.0.17 +version: 0.0.18 type: command is_deterministic: True @@ -9,7 +9,7 @@ display_name: Register model description: Register a model to a workspace or a registry. The component works on compute with [MSI](https://learn.microsoft.com/en-us/azure/machine-learning/how-to-create-manage-compute-instance?tabs=python) attached. -environment: azureml://registries/azureml/environments/python-sdk-v2/versions/21 +environment: azureml://registries/azureml/environments/python-sdk-v2/versions/22 code: ../../src command: >- diff --git a/assets/training/model_management/components/convert_model_to_mlflow/spec.yaml b/assets/training/model_management/components/convert_model_to_mlflow/spec.yaml index d5ebea4dfe..96433d90ac 100644 --- a/assets/training/model_management/components/convert_model_to_mlflow/spec.yaml +++ b/assets/training/model_management/components/convert_model_to_mlflow/spec.yaml @@ -1,7 +1,7 @@ $schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json name: convert_model_to_mlflow -version: 0.0.31 +version: 0.0.32 type: command is_deterministic: True @@ -9,7 +9,7 @@ is_deterministic: True display_name: Convert models to MLflow description: Component converts models from supported frameworks to MLflow model packaging format -environment: azureml://registries/azureml/environments/model-management/versions/32 +environment: azureml://registries/azureml/environments/model-management/versions/34 code: ../../src/ command: | diff --git a/assets/training/model_management/components/download_model/spec.yaml b/assets/training/model_management/components/download_model/spec.yaml index e86c20f94e..44934afba2 100644 --- a/assets/training/model_management/components/download_model/spec.yaml +++ b/assets/training/model_management/components/download_model/spec.yaml @@ -1,7 +1,7 @@ $schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json name: download_model -version: 0.0.28 +version: 0.0.29 type: command is_deterministic: True @@ -9,7 +9,7 @@ is_deterministic: True display_name: Download model description: Downloads a publicly available model -environment: azureml://registries/azureml/environments/model-management/versions/32 +environment: azureml://registries/azureml/environments/model-management/versions/34 code: ../../src/ command: > diff --git a/assets/training/model_management/components/import_model/spec.yaml b/assets/training/model_management/components/import_model/spec.yaml index 50798cd4e2..904154c4e3 100644 --- a/assets/training/model_management/components/import_model/spec.yaml +++ b/assets/training/model_management/components/import_model/spec.yaml @@ -4,7 +4,7 @@ type: pipeline name: import_model display_name: Import model description: Import a model into a workspace or a registry -version: 0.0.36 +version: 0.0.37 # Pipeline inputs inputs: @@ -207,7 +207,7 @@ outputs: jobs: validation_trigger_import: - component: azureml:validation_trigger_import:0.0.11 + component: azureml:validation_trigger_import:0.0.12 compute: ${{parent.inputs.compute}} resources: instance_type: '${{parent.inputs.instance_type}}' @@ -241,7 +241,7 @@ jobs: type: uri_file download_model: - component: azureml:download_model:0.0.28 + component: azureml:download_model:0.0.29 compute: ${{parent.inputs.compute}} resources: instance_type: '${{parent.inputs.instance_type}}' @@ -260,7 +260,7 @@ jobs: type: uri_folder convert_model_to_mlflow: - component: azureml:convert_model_to_mlflow:0.0.31 + component: azureml:convert_model_to_mlflow:0.0.32 compute: ${{parent.inputs.compute}} resources: instance_type: '${{parent.inputs.instance_type}}' @@ -287,7 +287,7 @@ jobs: type: mlflow_model mlflow_model_local_validation: - component: azureml:mlflow_model_local_validation:0.0.14 + component: azureml:mlflow_model_local_validation:0.0.15 compute: ${{parent.inputs.compute}} resources: instance_type: '${{parent.inputs.instance_type}}' @@ -300,7 +300,7 @@ jobs: mlflow_model_folder: ${{parent.outputs.mlflow_model_folder}} register_model: - component: azureml:register_model:0.0.17 + component: azureml:register_model:0.0.18 compute: ${{parent.inputs.compute}} resources: instance_type: '${{parent.inputs.instance_type}}' diff --git a/assets/training/model_management/components/validation_trigger_import/spec.yaml b/assets/training/model_management/components/validation_trigger_import/spec.yaml index 358550ce13..63c229eeea 100644 --- a/assets/training/model_management/components/validation_trigger_import/spec.yaml +++ b/assets/training/model_management/components/validation_trigger_import/spec.yaml @@ -4,7 +4,7 @@ type: command name: validation_trigger_import display_name: Validation Trigger Import model description: Component for enabling validation of import pipeline. -version: 0.0.11 +version: 0.0.12 # Pipeline inputs inputs: @@ -180,7 +180,7 @@ outputs: is_deterministic: True -environment: azureml://registries/azureml/environments/python-sdk-v2/versions/21 +environment: azureml://registries/azureml/environments/python-sdk-v2/versions/22 code: ../../src command: python run_model_validate.py --validation-info ${{outputs.validation_info}} From 9b88fc508235b2c2e50682da70d8d8aa9dfcca1e Mon Sep 17 00:00:00 2001 From: Kelly <40868256+lykelly19@users.noreply.github.com> Date: Tue, 24 Sep 2024 10:46:18 -0400 Subject: [PATCH 7/8] Allow data assets to use azcopy from external storage account & refactor models code (#3317) * Allow data assets to use azcopy form external storage account usage & refactor models code * update * rename from model_utils to registry_utils * some scripts-syntax updates * more scripts-syntax changes * more cleanup * Update CHANGELOG and setup.py * update spacing * syntax * move pluralize_asset_type to registry_utils * spacing * update import * rename function * spacing * update docstring * Update scripts/azureml-assets/CHANGELOG.md Co-authored-by: Louie Larson * Update types and condition check * update version * add space --------- Co-authored-by: lykelly19 Co-authored-by: Louie Larson --- scripts/azureml-assets/CHANGELOG.md | 4 + .../azureml-assets/azureml/assets/__init__.py | 1 + .../{model_utils.py => registry_utils.py} | 206 +++++++++++++----- .../azureml/assets/publish_utils.py | 58 +++-- scripts/azureml-assets/setup.py | 2 +- 5 files changed, 208 insertions(+), 63 deletions(-) rename scripts/azureml-assets/azureml/assets/model/{model_utils.py => registry_utils.py} (51%) diff --git a/scripts/azureml-assets/CHANGELOG.md b/scripts/azureml-assets/CHANGELOG.md index ed2bc448c8..98c34359bd 100644 --- a/scripts/azureml-assets/CHANGELOG.md +++ b/scripts/azureml-assets/CHANGELOG.md @@ -3,6 +3,10 @@ ### 🐛 Bugs Fixed +## 1.16.59 (2024-09-24) +### 🚀 New Features +- [#3317](https://github.com/Azure/azureml-assets/pull/3317) Support for storage account access for data asset copy with SAS token + ## 1.16.58 (2024-09-13) ### 🐛 Bugs Fixed - [#3377](https://github.com/Azure/azureml-assets/pull/3377) Validation for vision results diff --git a/scripts/azureml-assets/azureml/assets/__init__.py b/scripts/azureml-assets/azureml/assets/__init__.py index 0c441adcac..b6594e71c2 100644 --- a/scripts/azureml-assets/azureml/assets/__init__.py +++ b/scripts/azureml-assets/azureml/assets/__init__.py @@ -6,6 +6,7 @@ AssetType, ComponentType, Config, + DataConfig, DEFAULT_ASSET_FILENAME, ModelConfig, ModelFlavor, diff --git a/scripts/azureml-assets/azureml/assets/model/model_utils.py b/scripts/azureml-assets/azureml/assets/model/registry_utils.py similarity index 51% rename from scripts/azureml-assets/azureml/assets/model/model_utils.py rename to scripts/azureml-assets/azureml/assets/model/registry_utils.py index 899afa27dd..f5ba4df49e 100644 --- a/scripts/azureml-assets/azureml/assets/model/model_utils.py +++ b/scripts/azureml-assets/azureml/assets/model/registry_utils.py @@ -4,14 +4,16 @@ import copy import azureml.assets as assets -from typing import Tuple -from azure.ai.ml import load_model, MLClient +from typing import Tuple, Union +from azure.ai.ml import load_model, load_data, MLClient, operations as ops from azure.ai.ml._utils._registry_utils import get_asset_body_for_registry_storage from azureml.assets.util import logger +from azureml.assets.config import AssetType, Config, DataConfig, ModelConfig from azureml.assets.util.util import resolve_from_file_for_asset from azureml.assets.config import PathType from azureml.assets.model.download_utils import CopyUpdater, copy_azure_artifacts, download_git_model from azureml.assets.deployment_config import AssetVersionUpdate +from pathlib import Path class RegistryUtils: @@ -19,14 +21,87 @@ class RegistryUtils: RETRY_COUNT = 3 - def get_registry_data_reference(model_name: str, model_version: str, ml_client: MLClient) -> Tuple[str, str]: + def pluralize_asset_type(asset_type: Union[AssetType, str]) -> str: + """Return pluralized asset type.""" + # Convert to string if enum + if isinstance(asset_type, AssetType): + asset_type = asset_type.value + return f"{asset_type}s" if asset_type != "data" else asset_type + + def get_operations_from_type(asset_type: AssetType, ml_client: MLClient) -> Union[ + ops.ComponentOperations, ops.DataOperations, ops.EnvironmentOperations, + ops.ModelOperations]: + """Get MLCLient operations related to an asset type. + + Args: + asset_type (AssetType): Asset type. + ml_client (MLClient): ML client. + + Returns: + Union[ops.ComponentOperations, ops.DataOperations, ops.EnvironmentOperations, + ops.ModelOperations]: Operations object. + """ + if asset_type == AssetType.COMPONENT: + return ml_client.components + elif asset_type == AssetType.DATA: + return ml_client.data + elif asset_type == AssetType.ENVIRONMENT: + return ml_client.environments + elif asset_type == AssetType.MODEL: + return ml_client.models + + def publish_to_registry(ml_client: MLClient, extra_config: Config, asset_name: str, asset_version: str, + asset_type: assets.AssetType, temp_dir: Path, copy_updater: CopyUpdater = None): + """Copy artifacts to registry storage. + + Args: + ml_client (MLClient): ML client. + extra_config (Config): ModelConfig or DataConfig object containing external storage info. + asset_name (str): Asset name. + asset_version (str): Asset version. + asset_type (assets.AssetType): Asset type. + temp_dir (Path): temp dir for asset operation. + copy_updater (CopyUpdater): CopyUpdater object to update files during azcopy. + """ + src_uri = extra_config.path.uri + if extra_config.path.type == PathType.GIT: + # download model locally (this is supported for models) + src_uri = temp_dir + logger.print(f"Cloning {asset_type} files from git {extra_config.path.uri}") + success = download_git_model(extra_config.path.uri, src_uri) + logger.print("Completed cloning") + if not success: + raise Exception(f"Cloning uri {extra_config.path.uri} failed") + + try: + # copy to registry blobstorage (this is supported for models & data assets) + logger.print("get data ref for registry storage upload") + + blob_uri, sas_uri = RegistryUtils.get_registry_data_reference( + asset_name, asset_version, asset_type, ml_client + ) + + success = copy_azure_artifacts(src_uri, sas_uri, copy_updater) + if not success: + raise Exception("blobstorage copy failed.") + logger.print(f"Successfully copied {asset_type.value} artifacts to registry storage") + return blob_uri + except Exception as e: + raise Exception(f"Error in copying artifacts to registry storage. Error {e}") + + def get_registry_data_reference(asset_name: str, asset_version: str, asset_type: assets.AssetType, + ml_client: MLClient) -> Tuple[str, str]: """Fetch data reference for asset in the registry.""" - registry_name = ml_client.models._registry_name - asset_id = f"azureml://registries/{registry_name}/models/{model_name}/versions/{model_version}" + asset_type_pluralized = RegistryUtils.pluralize_asset_type(asset_type) + operations = RegistryUtils.get_operations_from_type(asset_type=asset_type, ml_client=ml_client) + registry_name = operations._registry_name + + asset_id = f"azureml://registries/{registry_name}/{asset_type_pluralized}/" \ + f"{asset_name}/versions/{asset_version}" logger.print(f"getting data reference for asset {asset_id}") for cnt in range(1, RegistryUtils.RETRY_COUNT + 1): try: - response = RegistryUtils._get_temp_data_ref(model_name, model_version, ml_client) + response = RegistryUtils._get_temp_data_ref(asset_name, asset_version, asset_type, ml_client) blob_uri = response.blob_reference_for_consumption.blob_uri sas_uri = response.blob_reference_for_consumption.credential.additional_properties["sasUri"] if not blob_uri or not sas_uri: @@ -37,31 +112,69 @@ def get_registry_data_reference(model_name: str, model_version: str, ml_client: else: raise Exception(f"Unable to fetch data reference for asset {asset_id}") - def _get_temp_data_ref(model_name: str, model_version: str, ml_client: MLClient): - service_client = ml_client.models._service_client - registry_name = ml_client.models._registry_name - body = get_asset_body_for_registry_storage(registry_name, "models", model_name, model_version) + def _get_temp_data_ref(asset_name: str, asset_version: str, asset_type: assets.AssetType, ml_client: MLClient): + asset_type_pluralized = RegistryUtils.pluralize_asset_type(asset_type) + operations = RegistryUtils.get_operations_from_type(asset_type=asset_type, ml_client=ml_client) + service_client = operations._service_client + resource_group_name = operations._resource_group_name + registry_name = operations._registry_name + + body = get_asset_body_for_registry_storage(registry_name, asset_type_pluralized, asset_name, asset_version) response = service_client.temporary_data_references.create_or_get_temporary_data_reference( - name=model_name, - version=model_version, - resource_group_name=ml_client.models._resource_group_name, + name=asset_name, + version=asset_version, + resource_group_name=resource_group_name, registry_name=registry_name, body=body, ) return response -class ModelAsset: - """Asset class for model.""" +class Asset: + """Asset class.""" - def __init__(self, spec_path, model_config, registry_name, temp_dir, copy_updater: CopyUpdater = None): - """Initialize model asset.""" + def __init__(self, spec_path: Path, extra_config: Config, registry_name: str, temp_dir: Path, + copy_updater: CopyUpdater = None): + """Initialize asset.""" self._spec_path = spec_path - self._model_config = model_config - self._registry_name = registry_name + self._extra_config = extra_config self._temp_dir = temp_dir self._copy_updater = copy_updater + +class DataAsset(Asset): + """Asset class for data.""" + + def __init__(self, spec_path: Path, data_config: DataConfig, registry_name: str, temp_dir: Path, + copy_updater: CopyUpdater = None): + """Initialize data asset.""" + super().__init__(spec_path, data_config, registry_name, temp_dir, copy_updater) + self._data_config = data_config + + try: + self._data = load_data(spec_path) + except Exception as e: + logger.error(f"Error in loading data spec file at {spec_path}: {e}") + return False + + def prepare_data(self, ml_client: MLClient): + """Prepare data for publish.""" + data_registry_path = RegistryUtils.publish_to_registry(ml_client, self._data_config, self._data.name, + self._data.version, AssetType.DATA, self._temp_dir, + self._copy_updater) + self._data.path = data_registry_path + return self._data + + +class ModelAsset(Asset): + """Asset class for model.""" + + def __init__(self, spec_path: Path, model_config: ModelConfig, registry_name: str, temp_dir: Path, + copy_updater: CopyUpdater = None): + """Initialize model asset.""" + super().__init__(spec_path, model_config, registry_name, temp_dir, copy_updater) + self._model_config = model_config + try: self._model = load_model(spec_path) self._model.description = model_config.description @@ -73,31 +186,6 @@ def __init__(self, spec_path, model_config, registry_name, temp_dir, copy_update logger.error(f"Error in loading model spec file at {spec_path}: {e}") return False - def _publish_to_registry(self, ml_client: MLClient, copy_updater: CopyUpdater = None): - src_uri = self._model_config.path.uri - if self._model_config.path.type == PathType.GIT: - # download model locally - src_uri = self._temp_dir - logger.print(f"Cloning model files from git {self._model_config.path.uri}") - success = download_git_model(self._model_config.path.uri, src_uri) - logger.print("Completed cloning") - if not success: - raise Exception(f"Cloning uri {self._model_config.path.uri} failed") - - try: - # copy to registry blobstorage - logger.print("get data ref for registry storage upload") - blob_uri, sas_uri = RegistryUtils.get_registry_data_reference( - self._model.name, self._model.version, ml_client - ) - success = copy_azure_artifacts(src_uri, sas_uri, copy_updater) - if not success: - raise Exception("blobstorage copy failed.") - logger.print("Successfully copied model artifacts to registry storage") - return blob_uri - except Exception as e: - raise Exception(f"Error in copying artifacts to registry storage. Error {e}") - class MLFlowModelAsset(ModelAsset): """Asset class for MLflow model.""" @@ -105,13 +193,16 @@ class MLFlowModelAsset(ModelAsset): MLMODEL_FILE_NAME = "MLmodel" MLFLOW_MODEL_PATH = "mlflow_model_folder" - def __init__(self, spec_path, model_config, registry_name, temp_dir, copy_updater: CopyUpdater = None): + def __init__(self, spec_path: Path, model_config: ModelConfig, registry_name: str, temp_dir: Path, + copy_updater: CopyUpdater = None): """Initialize Mlflow model asset.""" super().__init__(spec_path, model_config, registry_name, temp_dir, copy_updater) def prepare_model(self, ml_client: MLClient): """Prepare model for publish.""" - model_registry_path = self._publish_to_registry(ml_client, self._copy_updater) + model_registry_path = RegistryUtils.publish_to_registry(ml_client, self._model_config, self._model.name, + self._model.version, AssetType.MODEL, self._temp_dir, + self._copy_updater) self._model.path = model_registry_path + "/" + MLFlowModelAsset.MLFLOW_MODEL_PATH return self._model @@ -119,18 +210,22 @@ def prepare_model(self, ml_client: MLClient): class CustomModelAsset(ModelAsset): """Asset class for custom model.""" - def __init__(self, spec_path, model_config, registry_name, temp_dir, copy_updater: CopyUpdater = None): + def __init__(self, spec_path: Path, model_config: ModelConfig, registry_name: str, temp_dir: Path, + copy_updater: CopyUpdater = None): """Initialize custom model asset.""" super().__init__(spec_path, model_config, registry_name, temp_dir, copy_updater) def prepare_model(self, ml_client: MLClient): """Prepare model for publish.""" - model_registry_path = self._publish_to_registry(ml_client, self._copy_updater) + model_registry_path = RegistryUtils.publish_to_registry(ml_client, self._model_config, self._model.name, + self._model.version, AssetType.MODEL, self._temp_dir, + self._copy_updater) self._model.path = model_registry_path return self._model -def prepare_model(spec_path, model_config, temp_dir, ml_client: MLClient, copy_updater: CopyUpdater = None): +def prepare_model(spec_path: Path, model_config: ModelConfig, temp_dir: Path, ml_client: MLClient, + copy_updater: CopyUpdater = None): """Prepare model for publish.""" try: logger.print(f"Model type: {model_config.type}") @@ -149,6 +244,19 @@ def prepare_model(spec_path, model_config, temp_dir, ml_client: MLClient, copy_u return None, False +def prepare_data(spec_path: Path, data_config: DataConfig, temp_dir: Path, ml_client: MLClient, + copy_updater: CopyUpdater = None): + """Prepare data for publish.""" + try: + registry_name = ml_client.data._registry_name + data_asset = DataAsset(spec_path, data_config, registry_name, temp_dir, copy_updater) + data = data_asset.prepare_data(ml_client) + return data, True + except Exception as e: + logger.log_error(f"prepare data failed for {spec_path}. Error {e}") + return None, False + + def update_model_metadata( model_name: str, model_version: str, diff --git a/scripts/azureml-assets/azureml/assets/publish_utils.py b/scripts/azureml-assets/azureml/assets/publish_utils.py index a2cb91a436..466e6225d0 100644 --- a/scripts/azureml-assets/azureml/assets/publish_utils.py +++ b/scripts/azureml-assets/azureml/assets/publish_utils.py @@ -13,9 +13,10 @@ from subprocess import CompletedProcess, run from tempfile import TemporaryDirectory from typing import Dict, List, Tuple, Union -from azureml.assets.config import AssetConfig, AssetType, ComponentType, ModelConfig +from azureml.assets.config import AssetConfig, AssetType, ComponentType, ModelConfig, DataConfig from azureml.assets.deployment_config import AssetVersionUpdate -from azureml.assets.model.model_utils import CopyUpdater, prepare_model, update_model_metadata +from azureml.assets.model.registry_utils import CopyUpdater, prepare_model, update_model_metadata, \ + prepare_data, RegistryUtils from azureml.assets.util import logger from azureml.assets.util.util import resolve_from_file_for_asset from azure.ai.ml import MLClient @@ -31,14 +32,6 @@ LATEST_LABEL = "latest" -def pluralize_asset_type(asset_type: Union[AssetType, str]) -> str: - """Return pluralized asset type.""" - # Convert to string if enum - if isinstance(asset_type, AssetType): - asset_type = asset_type.value - return f"{asset_type}s" if asset_type != "data" else asset_type - - def sanitize_output(input: str) -> str: """Return sanitized string.""" # Remove sensitive token @@ -49,7 +42,7 @@ def sanitize_output(input: str) -> str: def update_spec(asset: Union[Component, Environment, Model], spec_path: Path) -> bool: """Update the yaml spec file with updated properties in asset. - :param asset: Asset loaded using load_*(component, environemnt, model) method. + :param asset: Asset loaded using load_*(component, environment, model) method. :type asset: Union[Component, Environment, Model] :param spec_path: path to asset spec file :type spec_path: Path @@ -97,6 +90,38 @@ def prepare_model_for_registration( return success +def prepare_data_for_registration( + data_config: DataConfig, + spec_file_path: Path, + temp_dir: Path, + ml_client: MLClient, + copy_updater: CopyUpdater = None, +) -> bool: + """Prepare data. + + :param data_config: Data Config object + :type data_config: DataConfig + :param spec_file_path: path to data spec file + :type spec_file_path: Path + :param temp_dir: temp dir for data operation + :type temp_dir: Path + :param ml_client: MLClient object + :type ml_client: MLClient + :param copy_updater: CopyUpdater object to update files during azcopy + :type copy_updater: CopyUpdater + :return: Data successfully prepared for creation in registry. + :rtype: bool + """ + data, success = prepare_data( + spec_path=spec_file_path, data_config=data_config, temp_dir=temp_dir, ml_client=ml_client, + copy_updater=copy_updater + ) + if success: + success = update_spec(data, spec_file_path) + logger.print(f"updated spec file? {success}") + return success + + def validate_and_prepare_pipeline_component( spec_path: Path, registry_name: str, @@ -420,7 +445,7 @@ def get_parsed_details_from_asset_uri(asset_type: str, asset_uri: str) -> Tuple[ :rtype: Tuple """ REGISTRY_ASSET_PATTERN = re.compile(REGISTRY_ASSET_TEMPLATE.substitute( - asset_type=pluralize_asset_type(asset_type))) + asset_type=RegistryUtils.pluralize_asset_type(asset_type))) asset_registry_name = None if (match := REGISTRY_ASSET_PATTERN.match(asset_uri)) is not None: asset_registry_name, asset_name, asset_version, asset_label = match.groups() @@ -523,11 +548,18 @@ def create_asset(asset: AssetConfig, registry_name: str, ml_client: MLClient, ve return False elif asset.type == AssetType.MODEL: version = asset.version - model_config = asset.extra_config_as_object() + model_config: ModelConfig = asset.extra_config_as_object() if not prepare_model_for_registration(model_config, asset.spec_with_path, Path(temp_dir), ml_client, copy_updater): logger.log_error("Failed to prepare model") return False + elif asset.type == AssetType.DATA: + version = asset.version + data_config: DataConfig = asset.extra_config_as_object() + if not prepare_data_for_registration(data_config, asset.spec_with_path, Path(temp_dir), ml_client, + copy_updater): + logger.log_error("Failed to prepare data asset") + return False # Create asset return create_asset_cli( diff --git a/scripts/azureml-assets/setup.py b/scripts/azureml-assets/setup.py index 3c3c2fd60c..d074107716 100644 --- a/scripts/azureml-assets/setup.py +++ b/scripts/azureml-assets/setup.py @@ -7,7 +7,7 @@ setup( name="azureml-assets", - version="1.16.58", + version="1.16.59", description="Utilities for publishing assets to Azure Machine Learning system registries.", author="Microsoft Corp", packages=find_packages(), From 5e290e4b61580dfd381ebdf9770bbee4d6320e35 Mon Sep 17 00:00:00 2001 From: Kelly <40868256+lykelly19@users.noreply.github.com> Date: Tue, 24 Sep 2024 11:00:01 -0400 Subject: [PATCH 8/8] Upgrade test resource python version to 3.10 (#3421) Co-authored-by: lykelly19 --- .../good-validation/context/conda_dependencies.yaml | 2 +- .../validate/good-validation/context/conda_dependencies.yaml | 2 +- .../image-name-mismatch/context/conda_dependencies.yaml | 2 +- .../missing-description-file/context/conda_dependencies.yaml | 2 +- .../publishing-disabled/context/conda_dependencies.yaml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/test/resources/validate-copyright/good-validation/context/conda_dependencies.yaml b/test/resources/validate-copyright/good-validation/context/conda_dependencies.yaml index 2dab1832cb..66ca274bfe 100644 --- a/test/resources/validate-copyright/good-validation/context/conda_dependencies.yaml +++ b/test/resources/validate-copyright/good-validation/context/conda_dependencies.yaml @@ -2,7 +2,7 @@ channels: - anaconda - conda-forge dependencies: -- python=3.8 +- python=3.10 - pip=20.2.4 - pandas~=1.3.0 - scipy~=1.7.0 diff --git a/test/resources/validate/good-validation/context/conda_dependencies.yaml b/test/resources/validate/good-validation/context/conda_dependencies.yaml index 2dab1832cb..66ca274bfe 100644 --- a/test/resources/validate/good-validation/context/conda_dependencies.yaml +++ b/test/resources/validate/good-validation/context/conda_dependencies.yaml @@ -2,7 +2,7 @@ channels: - anaconda - conda-forge dependencies: -- python=3.8 +- python=3.10 - pip=20.2.4 - pandas~=1.3.0 - scipy~=1.7.0 diff --git a/test/resources/validate/image-name-mismatch/context/conda_dependencies.yaml b/test/resources/validate/image-name-mismatch/context/conda_dependencies.yaml index 2dab1832cb..66ca274bfe 100644 --- a/test/resources/validate/image-name-mismatch/context/conda_dependencies.yaml +++ b/test/resources/validate/image-name-mismatch/context/conda_dependencies.yaml @@ -2,7 +2,7 @@ channels: - anaconda - conda-forge dependencies: -- python=3.8 +- python=3.10 - pip=20.2.4 - pandas~=1.3.0 - scipy~=1.7.0 diff --git a/test/resources/validate/missing-description-file/context/conda_dependencies.yaml b/test/resources/validate/missing-description-file/context/conda_dependencies.yaml index 2dab1832cb..66ca274bfe 100644 --- a/test/resources/validate/missing-description-file/context/conda_dependencies.yaml +++ b/test/resources/validate/missing-description-file/context/conda_dependencies.yaml @@ -2,7 +2,7 @@ channels: - anaconda - conda-forge dependencies: -- python=3.8 +- python=3.10 - pip=20.2.4 - pandas~=1.3.0 - scipy~=1.7.0 diff --git a/test/resources/validate/publishing-disabled/context/conda_dependencies.yaml b/test/resources/validate/publishing-disabled/context/conda_dependencies.yaml index 2dab1832cb..66ca274bfe 100644 --- a/test/resources/validate/publishing-disabled/context/conda_dependencies.yaml +++ b/test/resources/validate/publishing-disabled/context/conda_dependencies.yaml @@ -2,7 +2,7 @@ channels: - anaconda - conda-forge dependencies: -- python=3.8 +- python=3.10 - pip=20.2.4 - pandas~=1.3.0 - scipy~=1.7.0