From d02ca882bfa2ac9fa5530041b8ff655f47315562 Mon Sep 17 00:00:00 2001 From: VishalPatil1304 <115918339+Vishal-p117@users.noreply.github.com> Date: Mon, 10 Jun 2024 18:15:38 +0530 Subject: [PATCH 1/3] Max sequence length and oss components version updated (#3029) --- .../components_v2/data_import/common/spec.yaml | 2 +- .../components_v2/finetune/chat_completion/spec.yaml | 2 +- .../components_v2/finetune/text_generation/spec.yaml | 2 +- .../pipeline_components/chat_completion/spec.yaml | 4 ++-- .../pipeline_components/text_generation/spec.yaml | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/assets/training/finetune_acft_hf_nlp/components_v2/data_import/common/spec.yaml b/assets/training/finetune_acft_hf_nlp/components_v2/data_import/common/spec.yaml index c127b30a11..12056326ce 100644 --- a/assets/training/finetune_acft_hf_nlp/components_v2/data_import/common/spec.yaml +++ b/assets/training/finetune_acft_hf_nlp/components_v2/data_import/common/spec.yaml @@ -1,6 +1,6 @@ $schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json name: oss_text_generation_data_import -version: 0.0.15 +version: 0.0.16 type: command is_deterministic: True diff --git a/assets/training/finetune_acft_hf_nlp/components_v2/finetune/chat_completion/spec.yaml b/assets/training/finetune_acft_hf_nlp/components_v2/finetune/chat_completion/spec.yaml index 28e20ff219..046578d467 100644 --- a/assets/training/finetune_acft_hf_nlp/components_v2/finetune/chat_completion/spec.yaml +++ b/assets/training/finetune_acft_hf_nlp/components_v2/finetune/chat_completion/spec.yaml @@ -1,6 +1,6 @@ $schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json name: oss_chat_completion_finetune -version: 0.0.15 +version: 0.0.16 type: command is_deterministic: True diff --git a/assets/training/finetune_acft_hf_nlp/components_v2/finetune/text_generation/spec.yaml b/assets/training/finetune_acft_hf_nlp/components_v2/finetune/text_generation/spec.yaml index 73193616fa..6fb5c31c15 100644 --- a/assets/training/finetune_acft_hf_nlp/components_v2/finetune/text_generation/spec.yaml +++ b/assets/training/finetune_acft_hf_nlp/components_v2/finetune/text_generation/spec.yaml @@ -1,6 +1,6 @@ $schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json name: oss_text_generation_finetune -version: 0.0.15 +version: 0.0.16 type: command is_deterministic: True diff --git a/assets/training/finetune_acft_hf_nlp/components_v2/pipeline_components/chat_completion/spec.yaml b/assets/training/finetune_acft_hf_nlp/components_v2/pipeline_components/chat_completion/spec.yaml index 287835a44d..41383d044f 100644 --- a/assets/training/finetune_acft_hf_nlp/components_v2/pipeline_components/chat_completion/spec.yaml +++ b/assets/training/finetune_acft_hf_nlp/components_v2/pipeline_components/chat_completion/spec.yaml @@ -1,6 +1,6 @@ $schema: https://azuremlschemas.azureedge.net/latest/pipelineComponent.schema.json name: oss_chat_completion_pipeline -version: 0.0.15 +version: 0.0.16 type: pipeline display_name: OSS Chat Completion Pipeline description: FTaaS Pipeline component for chat completion @@ -153,7 +153,7 @@ jobs: dataset_input: '${{parent.jobs.oss_text_generation_data_import.outputs.output_dataset}}' batch_size: 1000 pad_to_max_length: "false" - max_seq_length: -1 + max_seq_length: 8192 number_of_gpu_to_use_finetuning: '${{parent.inputs.number_of_gpu_to_use_finetuning}}' apply_lora: "true" lora_alpha: 128 diff --git a/assets/training/finetune_acft_hf_nlp/components_v2/pipeline_components/text_generation/spec.yaml b/assets/training/finetune_acft_hf_nlp/components_v2/pipeline_components/text_generation/spec.yaml index 1f189c8005..b662a89ffd 100644 --- a/assets/training/finetune_acft_hf_nlp/components_v2/pipeline_components/text_generation/spec.yaml +++ b/assets/training/finetune_acft_hf_nlp/components_v2/pipeline_components/text_generation/spec.yaml @@ -1,6 +1,6 @@ $schema: https://azuremlschemas.azureedge.net/latest/pipelineComponent.schema.json name: oss_text_generation_pipeline -version: 0.0.15 +version: 0.0.16 type: pipeline display_name: OSS Text Generation Pipeline description: FTaaS Pipeline component for text generation From 62c4b62f911709de88bf2e480b5dda319afe295f Mon Sep 17 00:00:00 2001 From: Fred Li <51424245+fredms@users.noreply.github.com> Date: Mon, 10 Jun 2024 11:09:35 -0700 Subject: [PATCH 2/3] Upgrade RAG component version (#3030) --- .../data_ingestion_dataset_to_acs_basic/spec.yaml | 14 +++++++------- .../spec.yaml | 14 +++++++------- .../spec.yaml | 14 +++++++------- .../spec.yaml | 14 +++++++------- .../rag/components/crack_and_chunk/spec.yaml | 2 +- .../components/crack_and_chunk_and_embed/spec.yaml | 2 +- .../crack_chunk_embed_index_and_register/spec.yaml | 2 +- .../rag/components/crawl_url/spec.yaml | 2 +- .../rag/components/create_faiss_index/spec.yaml | 2 +- .../rag/components/create_promptflow/spec.yaml | 2 +- .../rag/components/data_import_acs/spec.yaml | 2 +- .../rag/components/generate_embeddings/spec.yaml | 2 +- .../generate_embeddings_parallel/spec.yaml | 2 +- .../rag/components/git_clone/spec.yaml | 2 +- .../rag/components/image_embed_index/spec.yaml | 2 +- .../rag/components/qa_data_generation/spec.yaml | 2 +- .../components/register_mlindex_asset/spec.yaml | 2 +- .../components/register_qa_data_asset/spec.yaml | 2 +- .../rag/components/update_acs_index/spec.yaml | 2 +- .../spec.yaml | 2 +- .../rag/components/update_milvus_index/spec.yaml | 2 +- .../rag/components/update_pinecone_index/spec.yaml | 2 +- .../rag/components/validate_deployments/spec.yaml | 2 +- 23 files changed, 47 insertions(+), 47 deletions(-) diff --git a/assets/large_language_models/components_pipelines/data_ingestion_dataset_to_acs_basic/spec.yaml b/assets/large_language_models/components_pipelines/data_ingestion_dataset_to_acs_basic/spec.yaml index 0f1295eb5f..f720bf75c7 100644 --- a/assets/large_language_models/components_pipelines/data_ingestion_dataset_to_acs_basic/spec.yaml +++ b/assets/large_language_models/components_pipelines/data_ingestion_dataset_to_acs_basic/spec.yaml @@ -4,7 +4,7 @@ type: pipeline tags: Preview: "" -version: 0.0.75 +version: 0.0.76 name: llm_ingest_dataset_to_acs_basic display_name: LLM - Dataset to ACS Pipeline is_deterministic: false @@ -110,7 +110,7 @@ jobs: properties: compute_specification: automatic: true - component: 'azureml:llm_rag_validate_deployments:0.0.66' + component: 'azureml:llm_rag_validate_deployments:0.0.68' identity: type: user_identity inputs: @@ -136,7 +136,7 @@ jobs: properties: compute_specification: automatic: true - component: 'azureml:llm_rag_crack_and_chunk:0.0.63' + component: 'azureml:llm_rag_crack_and_chunk:0.0.65' inputs: input_data: ${{parent.inputs.input_data}} input_glob: ${{parent.inputs.input_glob}} @@ -159,7 +159,7 @@ jobs: properties: compute_specification: automatic: true - component: 'azureml:llm_rag_create_promptflow:0.0.72' + component: 'azureml:llm_rag_create_promptflow:0.0.74' inputs: mlindex_asset_id: ${{parent.jobs.register_mlindex_asset_job.outputs.asset_id}} mlindex_asset_uri: ${{parent.jobs.create_acs_index_job.outputs.index}} @@ -177,7 +177,7 @@ jobs: properties: compute_specification: automatic: true - component: 'azureml:llm_rag_generate_embeddings:0.0.56' + component: 'azureml:llm_rag_generate_embeddings:0.0.58' inputs: chunks_source: type: uri_folder @@ -227,7 +227,7 @@ jobs: properties: compute_specification: automatic: true - component: 'azureml:llm_rag_update_acs_index:0.0.60' + component: 'azureml:llm_rag_update_acs_index:0.0.62' inputs: embeddings: type: uri_folder @@ -247,7 +247,7 @@ jobs: properties: compute_specification: automatic: true - component: 'azureml:llm_rag_register_mlindex_asset:0.0.60' + component: 'azureml:llm_rag_register_mlindex_asset:0.0.62' inputs: storage_uri: ${{parent.jobs.create_acs_index_job.outputs.index}} asset_name: ${{parent.inputs.embeddings_dataset_name}} diff --git a/assets/large_language_models/components_pipelines/data_ingestion_dataset_to_acs_user_id/spec.yaml b/assets/large_language_models/components_pipelines/data_ingestion_dataset_to_acs_user_id/spec.yaml index fc785d0bee..3bdd3fbb56 100644 --- a/assets/large_language_models/components_pipelines/data_ingestion_dataset_to_acs_user_id/spec.yaml +++ b/assets/large_language_models/components_pipelines/data_ingestion_dataset_to_acs_user_id/spec.yaml @@ -4,7 +4,7 @@ type: pipeline tags: Preview: "" -version: 0.0.73 +version: 0.0.74 name: llm_ingest_dataset_to_acs_user_id display_name: LLM - Dataset to ACS Pipeline is_deterministic: false @@ -110,7 +110,7 @@ jobs: properties: compute_specification: automatic: true - component: 'azureml:llm_rag_validate_deployments:0.0.66' + component: 'azureml:llm_rag_validate_deployments:0.0.68' identity: type: user_identity inputs: @@ -136,7 +136,7 @@ jobs: properties: compute_specification: automatic: true - component: 'azureml:llm_rag_crack_and_chunk:0.0.63' + component: 'azureml:llm_rag_crack_and_chunk:0.0.65' identity: type: user_identity inputs: @@ -161,7 +161,7 @@ jobs: properties: compute_specification: automatic: true - component: 'azureml:llm_rag_create_promptflow:0.0.72' + component: 'azureml:llm_rag_create_promptflow:0.0.74' identity: type: user_identity inputs: @@ -181,7 +181,7 @@ jobs: properties: compute_specification: automatic: true - component: 'azureml:llm_rag_generate_embeddings:0.0.56' + component: 'azureml:llm_rag_generate_embeddings:0.0.58' identity: type: user_identity inputs: @@ -235,7 +235,7 @@ jobs: properties: compute_specification: automatic: true - component: 'azureml:llm_rag_update_acs_index:0.0.60' + component: 'azureml:llm_rag_update_acs_index:0.0.62' identity: type: user_identity inputs: @@ -257,7 +257,7 @@ jobs: properties: compute_specification: automatic: true - component: 'azureml:llm_rag_register_mlindex_asset:0.0.60' + component: 'azureml:llm_rag_register_mlindex_asset:0.0.62' identity: type: user_identity inputs: diff --git a/assets/large_language_models/components_pipelines/data_ingestion_dataset_to_faiss_basic/spec.yaml b/assets/large_language_models/components_pipelines/data_ingestion_dataset_to_faiss_basic/spec.yaml index 3899b2ff46..c848a1afd1 100644 --- a/assets/large_language_models/components_pipelines/data_ingestion_dataset_to_faiss_basic/spec.yaml +++ b/assets/large_language_models/components_pipelines/data_ingestion_dataset_to_faiss_basic/spec.yaml @@ -4,7 +4,7 @@ type: pipeline tags: Preview: "" -version: 0.0.74 +version: 0.0.75 name: llm_ingest_dataset_to_faiss_basic display_name: LLM - Dataset to FAISS Pipeline is_deterministic: false @@ -102,7 +102,7 @@ jobs: properties: compute_specification: automatic: true - component: 'azureml:llm_rag_validate_deployments:0.0.66' + component: 'azureml:llm_rag_validate_deployments:0.0.68' identity: type: user_identity inputs: @@ -125,7 +125,7 @@ jobs: properties: compute_specification: automatic: true - component: 'azureml:llm_rag_crack_and_chunk:0.0.63' + component: 'azureml:llm_rag_crack_and_chunk:0.0.65' inputs: input_data: ${{parent.inputs.input_data}} input_glob: ${{parent.inputs.input_glob}} @@ -148,7 +148,7 @@ jobs: properties: compute_specification: automatic: true - component: 'azureml:llm_rag_create_promptflow:0.0.72' + component: 'azureml:llm_rag_create_promptflow:0.0.74' inputs: mlindex_asset_id: ${{parent.jobs.register_mlindex_asset_job.outputs.asset_id}} mlindex_asset_uri: ${{parent.jobs.create_faiss_index_job.outputs.index}} @@ -166,7 +166,7 @@ jobs: properties: compute_specification: automatic: true - component: 'azureml:llm_rag_generate_embeddings:0.0.56' + component: 'azureml:llm_rag_generate_embeddings:0.0.58' inputs: chunks_source: type: uri_folder @@ -216,7 +216,7 @@ jobs: properties: compute_specification: automatic: true - component: 'azureml:llm_rag_create_faiss_index:0.0.61' + component: 'azureml:llm_rag_create_faiss_index:0.0.63' inputs: embeddings: type: uri_folder @@ -234,7 +234,7 @@ jobs: properties: compute_specification: automatic: true - component: 'azureml:llm_rag_register_mlindex_asset:0.0.60' + component: 'azureml:llm_rag_register_mlindex_asset:0.0.62' inputs: storage_uri: ${{parent.jobs.create_faiss_index_job.outputs.index}} asset_name: ${{parent.inputs.embeddings_dataset_name}} diff --git a/assets/large_language_models/components_pipelines/data_ingestion_dataset_to_faiss_user_id/spec.yaml b/assets/large_language_models/components_pipelines/data_ingestion_dataset_to_faiss_user_id/spec.yaml index f4ccfd96d1..3b876ba73a 100644 --- a/assets/large_language_models/components_pipelines/data_ingestion_dataset_to_faiss_user_id/spec.yaml +++ b/assets/large_language_models/components_pipelines/data_ingestion_dataset_to_faiss_user_id/spec.yaml @@ -4,7 +4,7 @@ type: pipeline tags: Preview: "" -version: 0.0.73 +version: 0.0.74 name: llm_ingest_dataset_to_faiss_user_id display_name: LLM - Dataset to FAISS Pipeline is_deterministic: false @@ -102,7 +102,7 @@ jobs: properties: compute_specification: automatic: true - component: 'azureml:llm_rag_validate_deployments:0.0.66' + component: 'azureml:llm_rag_validate_deployments:0.0.68' identity: type: user_identity inputs: @@ -125,7 +125,7 @@ jobs: properties: compute_specification: automatic: true - component: 'azureml:llm_rag_crack_and_chunk:0.0.63' + component: 'azureml:llm_rag_crack_and_chunk:0.0.65' identity: type: user_identity inputs: @@ -150,7 +150,7 @@ jobs: properties: compute_specification: automatic: true - component: 'azureml:llm_rag_create_promptflow:0.0.72' + component: 'azureml:llm_rag_create_promptflow:0.0.74' identity: type: user_identity inputs: @@ -170,7 +170,7 @@ jobs: properties: compute_specification: automatic: true - component: 'azureml:llm_rag_generate_embeddings:0.0.56' + component: 'azureml:llm_rag_generate_embeddings:0.0.58' identity: type: user_identity inputs: @@ -224,7 +224,7 @@ jobs: properties: compute_specification: automatic: true - component: 'azureml:llm_rag_create_faiss_index:0.0.61' + component: 'azureml:llm_rag_create_faiss_index:0.0.63' identity: type: user_identity inputs: @@ -244,7 +244,7 @@ jobs: properties: compute_specification: automatic: true - component: 'azureml:llm_rag_register_mlindex_asset:0.0.60' + component: 'azureml:llm_rag_register_mlindex_asset:0.0.62' identity: type: user_identity inputs: diff --git a/assets/large_language_models/rag/components/crack_and_chunk/spec.yaml b/assets/large_language_models/rag/components/crack_and_chunk/spec.yaml index 8900864273..9a5b4c04fa 100644 --- a/assets/large_language_models/rag/components/crack_and_chunk/spec.yaml +++ b/assets/large_language_models/rag/components/crack_and_chunk/spec.yaml @@ -4,7 +4,7 @@ type: command tags: Preview: "" -version: 0.0.64 +version: 0.0.65 name: llm_rag_crack_and_chunk display_name: LLM - Crack and Chunk Data is_deterministic: true diff --git a/assets/large_language_models/rag/components/crack_and_chunk_and_embed/spec.yaml b/assets/large_language_models/rag/components/crack_and_chunk_and_embed/spec.yaml index ccb235d1d4..51af99056e 100644 --- a/assets/large_language_models/rag/components/crack_and_chunk_and_embed/spec.yaml +++ b/assets/large_language_models/rag/components/crack_and_chunk_and_embed/spec.yaml @@ -4,7 +4,7 @@ type: command tags: Preview: "" -version: 0.0.35 +version: 0.0.36 name: llm_rag_crack_and_chunk_and_embed display_name: LLM - Crack, Chunk and Embed Data is_deterministic: true diff --git a/assets/large_language_models/rag/components/crack_chunk_embed_index_and_register/spec.yaml b/assets/large_language_models/rag/components/crack_chunk_embed_index_and_register/spec.yaml index 950fa32787..65f3bf3797 100644 --- a/assets/large_language_models/rag/components/crack_chunk_embed_index_and_register/spec.yaml +++ b/assets/large_language_models/rag/components/crack_chunk_embed_index_and_register/spec.yaml @@ -1,5 +1,5 @@ name: llm_rag_crack_chunk_embed_index_and_register -version: 0.0.23 +version: 0.0.24 tags: Preview: "" diff --git a/assets/large_language_models/rag/components/crawl_url/spec.yaml b/assets/large_language_models/rag/components/crawl_url/spec.yaml index 3442a84687..556224fc67 100644 --- a/assets/large_language_models/rag/components/crawl_url/spec.yaml +++ b/assets/large_language_models/rag/components/crawl_url/spec.yaml @@ -4,7 +4,7 @@ type: command tags: Preview: "" -version: 0.0.22 +version: 0.0.23 name: llm_rag_crawl_url display_name: LLM - Crawl URL to Retrieve Data is_deterministic: true diff --git a/assets/large_language_models/rag/components/create_faiss_index/spec.yaml b/assets/large_language_models/rag/components/create_faiss_index/spec.yaml index 466b3110ae..6174f5bed3 100644 --- a/assets/large_language_models/rag/components/create_faiss_index/spec.yaml +++ b/assets/large_language_models/rag/components/create_faiss_index/spec.yaml @@ -4,7 +4,7 @@ type: command tags: Preview: "" -version: 0.0.62 +version: 0.0.63 name: llm_rag_create_faiss_index display_name: LLM - Create FAISS Index is_deterministic: true diff --git a/assets/large_language_models/rag/components/create_promptflow/spec.yaml b/assets/large_language_models/rag/components/create_promptflow/spec.yaml index 138137fec0..d0ef952852 100644 --- a/assets/large_language_models/rag/components/create_promptflow/spec.yaml +++ b/assets/large_language_models/rag/components/create_promptflow/spec.yaml @@ -4,7 +4,7 @@ type: command tags: Preview: "" -version: 0.0.73 +version: 0.0.74 name: llm_rag_create_promptflow display_name: LLM - Create Prompt Flow is_deterministic: true diff --git a/assets/large_language_models/rag/components/data_import_acs/spec.yaml b/assets/large_language_models/rag/components/data_import_acs/spec.yaml index 3f3e2f82bd..c8fb7165ed 100644 --- a/assets/large_language_models/rag/components/data_import_acs/spec.yaml +++ b/assets/large_language_models/rag/components/data_import_acs/spec.yaml @@ -4,7 +4,7 @@ type: command tags: Preview: "" -version: 0.0.58 +version: 0.0.59 name: llm_rag_data_import_acs display_name: LLM - Import Data from ACS is_deterministic: false diff --git a/assets/large_language_models/rag/components/generate_embeddings/spec.yaml b/assets/large_language_models/rag/components/generate_embeddings/spec.yaml index ac6f4b7bc2..73062693dc 100644 --- a/assets/large_language_models/rag/components/generate_embeddings/spec.yaml +++ b/assets/large_language_models/rag/components/generate_embeddings/spec.yaml @@ -4,7 +4,7 @@ type: command tags: Preview: "" -version: 0.0.57 +version: 0.0.58 name: llm_rag_generate_embeddings display_name: LLM - Generate Embeddings is_deterministic: true diff --git a/assets/large_language_models/rag/components/generate_embeddings_parallel/spec.yaml b/assets/large_language_models/rag/components/generate_embeddings_parallel/spec.yaml index de4084f4c0..b869968e0a 100644 --- a/assets/large_language_models/rag/components/generate_embeddings_parallel/spec.yaml +++ b/assets/large_language_models/rag/components/generate_embeddings_parallel/spec.yaml @@ -4,7 +4,7 @@ type: parallel tags: Preview: "" -version: 0.0.63 +version: 0.0.64 name: llm_rag_generate_embeddings_parallel display_name: LLM - Generate Embeddings Parallel is_deterministic: true diff --git a/assets/large_language_models/rag/components/git_clone/spec.yaml b/assets/large_language_models/rag/components/git_clone/spec.yaml index 78bbebf0c1..7e56dc97fb 100644 --- a/assets/large_language_models/rag/components/git_clone/spec.yaml +++ b/assets/large_language_models/rag/components/git_clone/spec.yaml @@ -4,7 +4,7 @@ type: command tags: Preview: "" -version: 0.0.61 +version: 0.0.62 name: llm_rag_git_clone display_name: LLM - Clone Git Repo is_deterministic: true diff --git a/assets/large_language_models/rag/components/image_embed_index/spec.yaml b/assets/large_language_models/rag/components/image_embed_index/spec.yaml index 3095aceaee..844ef81cef 100644 --- a/assets/large_language_models/rag/components/image_embed_index/spec.yaml +++ b/assets/large_language_models/rag/components/image_embed_index/spec.yaml @@ -4,7 +4,7 @@ type: command tags: Preview: "" -version: 0.0.15 +version: 0.0.16 name: llm_rag_image_embed_index display_name: LLM - Embedding images with Florence is_deterministic: true diff --git a/assets/large_language_models/rag/components/qa_data_generation/spec.yaml b/assets/large_language_models/rag/components/qa_data_generation/spec.yaml index 1f3fc276c7..ff218c8212 100644 --- a/assets/large_language_models/rag/components/qa_data_generation/spec.yaml +++ b/assets/large_language_models/rag/components/qa_data_generation/spec.yaml @@ -4,7 +4,7 @@ type: command tags: Preview: "" -version: 0.0.61 +version: 0.0.62 name: llm_rag_qa_data_generation display_name: LLM - Generate QnA Test Data is_deterministic: true diff --git a/assets/large_language_models/rag/components/register_mlindex_asset/spec.yaml b/assets/large_language_models/rag/components/register_mlindex_asset/spec.yaml index 53bcd390f6..f3d7c17c40 100644 --- a/assets/large_language_models/rag/components/register_mlindex_asset/spec.yaml +++ b/assets/large_language_models/rag/components/register_mlindex_asset/spec.yaml @@ -4,7 +4,7 @@ type: command tags: Preview: "" -version: 0.0.61 +version: 0.0.62 name: llm_rag_register_mlindex_asset display_name: LLM - Register Vector Index Asset is_deterministic: true diff --git a/assets/large_language_models/rag/components/register_qa_data_asset/spec.yaml b/assets/large_language_models/rag/components/register_qa_data_asset/spec.yaml index 3d77b3e6f6..99b7312ab3 100644 --- a/assets/large_language_models/rag/components/register_qa_data_asset/spec.yaml +++ b/assets/large_language_models/rag/components/register_qa_data_asset/spec.yaml @@ -4,7 +4,7 @@ type: command tags: Preview: "" -version: 0.0.54 +version: 0.0.55 name: llm_rag_register_qa_data_asset display_name: LLM - Register QA Generation Data Asset is_deterministic: true diff --git a/assets/large_language_models/rag/components/update_acs_index/spec.yaml b/assets/large_language_models/rag/components/update_acs_index/spec.yaml index 287fbe6205..1476dd1ab0 100644 --- a/assets/large_language_models/rag/components/update_acs_index/spec.yaml +++ b/assets/large_language_models/rag/components/update_acs_index/spec.yaml @@ -4,7 +4,7 @@ type: command tags: Preview: "" -version: 0.0.61 +version: 0.0.62 name: llm_rag_update_acs_index display_name: LLM - Update ACS Index is_deterministic: true diff --git a/assets/large_language_models/rag/components/update_azure_cosmos_mongo_vcore_index/spec.yaml b/assets/large_language_models/rag/components/update_azure_cosmos_mongo_vcore_index/spec.yaml index af9078c6e2..218ec3d18e 100644 --- a/assets/large_language_models/rag/components/update_azure_cosmos_mongo_vcore_index/spec.yaml +++ b/assets/large_language_models/rag/components/update_azure_cosmos_mongo_vcore_index/spec.yaml @@ -4,7 +4,7 @@ type: command tags: Preview: "" -version: 0.0.17 +version: 0.0.18 name: llm_rag_update_cosmos_mongo_vcore_index display_name: LLM - Update Azure Cosmos Mongo vCore Index is_deterministic: true diff --git a/assets/large_language_models/rag/components/update_milvus_index/spec.yaml b/assets/large_language_models/rag/components/update_milvus_index/spec.yaml index 15f1861a92..f3dfeaba28 100644 --- a/assets/large_language_models/rag/components/update_milvus_index/spec.yaml +++ b/assets/large_language_models/rag/components/update_milvus_index/spec.yaml @@ -4,7 +4,7 @@ type: command tags: Preview: "" -version: 0.0.17 +version: 0.0.18 name: llm_rag_update_milvus_index display_name: LLM - Update Milvus Index is_deterministic: true diff --git a/assets/large_language_models/rag/components/update_pinecone_index/spec.yaml b/assets/large_language_models/rag/components/update_pinecone_index/spec.yaml index 4a1c2ef803..00e9d72d9e 100644 --- a/assets/large_language_models/rag/components/update_pinecone_index/spec.yaml +++ b/assets/large_language_models/rag/components/update_pinecone_index/spec.yaml @@ -4,7 +4,7 @@ type: command tags: Preview: "" -version: 0.0.29 +version: 0.0.30 name: llm_rag_update_pinecone_index display_name: LLM - Update Pinecone Index is_deterministic: true diff --git a/assets/large_language_models/rag/components/validate_deployments/spec.yaml b/assets/large_language_models/rag/components/validate_deployments/spec.yaml index 9f710590a6..ade5cd4518 100644 --- a/assets/large_language_models/rag/components/validate_deployments/spec.yaml +++ b/assets/large_language_models/rag/components/validate_deployments/spec.yaml @@ -4,7 +4,7 @@ type: command tags: Preview: "" -version: 0.0.67 +version: 0.0.68 name: llm_rag_validate_deployments display_name: LLM - Validate Deployments is_deterministic: false From dfff4a8264006e0a8b720359215221af5ae29c88 Mon Sep 17 00:00:00 2001 From: rdondera-microsoft <98922913+rdondera-microsoft@users.noreply.github.com> Date: Mon, 10 Jun 2024 13:27:19 -0700 Subject: [PATCH 3/3] Add torchvision to image generation environments (#3017) * Hack environment for image generation models. * Bring back save_image(). * Fix import for save_image() and add unit test. * Delint. * Add comment to unit test function. * Delint. --- .../mgmt/processors/common/vision_utils.py | 16 +++++++++++----- .../pyfunc/text_to_image/conda.yaml | 1 + .../tests/unittests/test_vision_utils.py | 19 ++++++++++++++++++- 3 files changed, 30 insertions(+), 6 deletions(-) diff --git a/assets/training/model_management/src/azureml/model/mgmt/processors/common/vision_utils.py b/assets/training/model_management/src/azureml/model/mgmt/processors/common/vision_utils.py index 795c3d1810..a0af9f0d17 100644 --- a/assets/training/model_management/src/azureml/model/mgmt/processors/common/vision_utils.py +++ b/assets/training/model_management/src/azureml/model/mgmt/processors/common/vision_utils.py @@ -3,17 +3,20 @@ """Helper utils for vision Mlflow models.""" -import logging -import PIL -import pandas as pd import base64 import io +import logging +import os import re import requests -import torch -from ast import literal_eval +import uuid + +import PIL +import pandas as pd import numpy as np +import torch +from ast import literal_eval from PIL import Image, UnidentifiedImageError from typing import Union @@ -23,9 +26,11 @@ # Uncomment the following line for mlflow debug mode # logging.getLogger("mlflow").setLevel(logging.DEBUG) + def save_image(output_folder: str, img: PIL.Image.Image, format: str) -> str: """ Save image in a folder designated for batch output and return image file path. + :param output_folder: directory path where we need to save files :type output_folder: str :param img: image object @@ -39,6 +44,7 @@ def save_image(output_folder: str, img: PIL.Image.Image, format: str) -> str: img.save(os.path.join(output_folder, filename), format=format) return filename + def get_pil_image(image: bytes) -> PIL.Image.Image: """ Convert image bytes to PIL image. diff --git a/assets/training/model_management/src/azureml/model/mgmt/processors/pyfunc/text_to_image/conda.yaml b/assets/training/model_management/src/azureml/model/mgmt/processors/pyfunc/text_to_image/conda.yaml index 62cc832a07..cdff80416c 100644 --- a/assets/training/model_management/src/azureml/model/mgmt/processors/pyfunc/text_to_image/conda.yaml +++ b/assets/training/model_management/src/azureml/model/mgmt/processors/pyfunc/text_to_image/conda.yaml @@ -8,6 +8,7 @@ dependencies: - pip: - mlflow==2.12.1 - torch==1.13.1 + - torchvision==0.14.1 - transformers==4.33.2 - diffusers==0.23.0 - accelerate==0.22.0 diff --git a/assets/training/model_management/tests/unittests/test_vision_utils.py b/assets/training/model_management/tests/unittests/test_vision_utils.py index 5e89b62719..7e05e413b2 100644 --- a/assets/training/model_management/tests/unittests/test_vision_utils.py +++ b/assets/training/model_management/tests/unittests/test_vision_utils.py @@ -4,19 +4,36 @@ """Test vision utility code.""" import base64 +import os import pytest import requests +import tempfile import pandas as pd from unittest.mock import patch +from PIL import Image, ImageChops -from azureml.model.mgmt.processors.common.vision_utils import process_image, process_image_pandas_series, _is_valid_url +from azureml.model.mgmt.processors.common.vision_utils import ( + process_image, process_image_pandas_series, save_image, _is_valid_url +) class TestVisionUtilities: """Test vision utility functions.""" + def test_save_image(self): + """Test that save_image() behaves correctly for png files.""" + image = Image.new("RGB", (640, 480)) + image.putpixel((320, 240), (0, 0, 255)) + image.putpixel((320, 241), (255, 255, 0)) + + with tempfile.TemporaryDirectory() as temporary_directory_name: + image_file_name = save_image(temporary_directory_name, image, "png") + + image2 = Image.open(os.path.join(temporary_directory_name, image_file_name)) + assert not ImageChops.difference(image, image2).getbbox() + def test_process_image_bytes(self): """Test process_image() with a bytes object.""" image_bytes = bytes([0, 1])