From db309d52c64d75e8f2a67ff12da2ac8d8aa210ad Mon Sep 17 00:00:00 2001 From: Ilya Matiach Date: Tue, 25 Feb 2025 15:23:44 -0500 Subject: [PATCH] Update RAI vision and text images to ubuntu 22.04 --- .github/workflows/assets-test.yaml | 10 ++--- .../responsibleai-text/context/Dockerfile | 28 ++++++++------ .../context/conda_dependencies.yaml | 1 - .../responsibleai-text/tests/requirements.txt | 4 +- .../responsibleai-vision/context/Dockerfile | 38 +++++++++++-------- .../context/conda_dependencies.yaml | 1 - .../tests/requirements.txt | 4 +- scripts/test/create_asset_test_matrix.py | 1 + 8 files changed, 48 insertions(+), 39 deletions(-) diff --git a/.github/workflows/assets-test.yaml b/.github/workflows/assets-test.yaml index 81003ab68b..e17ff714c5 100644 --- a/.github/workflows/assets-test.yaml +++ b/.github/workflows/assets-test.yaml @@ -161,8 +161,8 @@ jobs: uses: actions/upload-artifact@v4 if: always() with: - name: ${{ env.pytest_reports }} - path: ${{ env.pytest_reports }} + name: ${{ env.pytest_reports }}-${{ matrix.asset_config_name }} + path: ${{ env.pytest_reports }}-${{ matrix.asset_config_name }} report: name: Publish test results @@ -181,8 +181,8 @@ jobs: id: download-artifact uses: actions/download-artifact@v4 with: - name: ${{ env.pytest_reports }} - path: ${{ env.pytest_reports }} + name: ${{ env.pytest_reports }}-${{ matrix.asset_config_name }} + path: ${{ env.pytest_reports }}-${{ matrix.asset_config_name }} continue-on-error: true - name: Publish test results @@ -190,4 +190,4 @@ jobs: uses: EnricoMi/publish-unit-test-result-action@v2 with: check_name: Test Results for ${{ github.workflow }} - junit_files: ${{ env.pytest_reports }}/**/*.xml + junit_files: ${{ env.pytest_reports }}-${{ matrix.asset_config_name }}/**/*.xml diff --git a/assets/responsibleai/environments/responsibleai-text/context/Dockerfile b/assets/responsibleai/environments/responsibleai-text/context/Dockerfile index 2e3824a9e1..288245eb03 100644 --- a/assets/responsibleai/environments/responsibleai-text/context/Dockerfile +++ b/assets/responsibleai/environments/responsibleai-text/context/Dockerfile @@ -1,20 +1,18 @@ -FROM mcr.microsoft.com/azureml/openmpi4.1.0-ubuntu20.04:{{latest-image-tag}} +FROM mcr.microsoft.com/azureml/openmpi4.1.0-ubuntu22.04:{{latest-image-tag}} -ENV AZUREML_CONDA_ENVIRONMENT_PATH /azureml-envs/responsibleai-text - -# Prepend path to AzureML conda environment -ENV PATH $AZUREML_CONDA_ENVIRONMENT_PATH/bin:$PATH +WORKDIR / +ENV CONDA_PREFIX=/azureml-envs/responsibleai-text +ENV CONDA_DEFAULT_ENV=$CONDA_PREFIX # Create conda environment COPY conda_dependencies.yaml . -RUN conda env create -p $AZUREML_CONDA_ENVIRONMENT_PATH -f conda_dependencies.yaml -q && \ - rm conda_dependencies.yaml && \ - conda run -p $AZUREML_CONDA_ENVIRONMENT_PATH pip cache purge && \ - conda clean -a -y +RUN conda env create -p $CONDA_PREFIX -f conda_dependencies.yaml -q && \ + rm conda_dependencies.yaml + +# Prepend path to AzureML conda environment +ENV PATH=$CONDA_PREFIX/bin:$PATH RUN conda list -# Conda install and pip install could happen side by side. Remove crypytography with vulnerability from conda -RUN conda remove cryptography RUN pip install --pre 'azure-ai-ml' 'azure-storage-blob<=12.13.0' 'numpy==1.26.2' @@ -48,7 +46,13 @@ RUN pip install git+https://github.com/huggingface/evaluate.git RUN pip install 'Werkzeug>=3.0.3' RUN pip install 'tqdm>=4.66.3' +# clean conda and pip caches +RUN conda run -p $CONDA_PREFIX pip cache purge && \ + conda clean -a -y + +RUN rm -rf ~/.cache/pip + RUN pip freeze # This is needed for mpi to locate libpython -ENV LD_LIBRARY_PATH $AZUREML_CONDA_ENVIRONMENT_PATH/lib:$LD_LIBRARY_PATH +ENV LD_LIBRARY_PATH $CONDA_PREFIX/lib:$LD_LIBRARY_PATH diff --git a/assets/responsibleai/environments/responsibleai-text/context/conda_dependencies.yaml b/assets/responsibleai/environments/responsibleai-text/context/conda_dependencies.yaml index 29b984913f..6a7a10ab45 100644 --- a/assets/responsibleai/environments/responsibleai-text/context/conda_dependencies.yaml +++ b/assets/responsibleai/environments/responsibleai-text/context/conda_dependencies.yaml @@ -1,7 +1,6 @@ name: responsibleai-text channels: - conda-forge - - defaults - huggingface - anaconda - pytorch diff --git a/assets/responsibleai/environments/responsibleai-text/tests/requirements.txt b/assets/responsibleai/environments/responsibleai-text/tests/requirements.txt index cf979ad62f..5489153db4 100644 --- a/assets/responsibleai/environments/responsibleai-text/tests/requirements.txt +++ b/assets/responsibleai/environments/responsibleai-text/tests/requirements.txt @@ -1,3 +1,3 @@ -azure-ai-ml==1.22.4 -azure.identity==1.19.0 +azure-ai-ml==1.25.0 +azure.identity==1.20.0 azureml-core>=1.58.0 \ No newline at end of file diff --git a/assets/responsibleai/environments/responsibleai-vision/context/Dockerfile b/assets/responsibleai/environments/responsibleai-vision/context/Dockerfile index 6e191c3ae0..2cac53907a 100644 --- a/assets/responsibleai/environments/responsibleai-vision/context/Dockerfile +++ b/assets/responsibleai/environments/responsibleai-vision/context/Dockerfile @@ -1,25 +1,24 @@ -FROM mcr.microsoft.com/azureml/openmpi4.1.0-ubuntu20.04:{{latest-image-tag}} +FROM mcr.microsoft.com/azureml/openmpi4.1.0-ubuntu22.04:{{latest-image-tag}} + # Install OpenCV native dependencies RUN apt-get update RUN apt-get install -y python3-opencv RUN apt-get remove -y libavutil56 libswresample3 libavformat58 -RUN apt-get remove -y libavcodec58 libswscale5 libopenexr24 libpmix2 - -ENV AZUREML_CONDA_ENVIRONMENT_PATH /azureml-envs/responsibleai-vision +RUN apt-get remove -y libavcodec58 libswscale5 libpmix2 -# Prepend path to AzureML conda environment -ENV PATH $AZUREML_CONDA_ENVIRONMENT_PATH/bin:$PATH +WORKDIR / +ENV CONDA_PREFIX=/azureml-envs/responsibleai-vision +ENV CONDA_DEFAULT_ENV=$CONDA_PREFIX # Create conda environment COPY conda_dependencies.yaml . -RUN conda env create -p $AZUREML_CONDA_ENVIRONMENT_PATH -f conda_dependencies.yaml -q && \ - rm conda_dependencies.yaml && \ - conda run -p $AZUREML_CONDA_ENVIRONMENT_PATH pip cache purge && \ - conda clean -a -y +RUN conda env create -p $CONDA_PREFIX -f conda_dependencies.yaml -q && \ + rm conda_dependencies.yaml + +# Prepend path to AzureML conda environment +ENV PATH=$CONDA_PREFIX/bin:$PATH RUN conda list -# Conda install and pip install could happen side by side. Remove crypytography with vulnerability from conda -RUN conda remove cryptography RUN pip install --pre 'azure-ai-ml' 'azure-storage-blob<=12.13.0' @@ -38,8 +37,7 @@ RUN pip install 'azureml-core=={{latest-pypi-version}}' \ RUN pip install 'azureml-dataset-runtime=={{latest-pypi-version}}' \ 'azureml-automl-dnn-vision=={{latest-pypi-version}}' -RUN pip install 'shap==0.41.0' \ - 'scikit-learn>=1.5.1' \ +RUN pip install 'scikit-learn>=1.5.1' \ 'interpret-community==0.31.0' \ 'Pillow>=10.3.0' \ 'setuptools>=65.5.1' \ @@ -48,6 +46,8 @@ RUN pip install 'shap==0.41.0' \ 'statsmodels==0.14.0' \ 'gunicorn>=22.0.0' +RUN pip install 'shap==0.46.0' + # azureml-dataset-runtime[fuse] upper bound for pyarrow is 11.0.0 # so we install pyarrow in extra step to avoid conflict RUN pip install 'pyarrow>=14.0.1' @@ -62,10 +62,16 @@ RUN pip install 'Werkzeug>=3.0.3' \ # To resolve CVE-2024-5480 vulnerability issue for torch < 2.2.2 RUN pip install 'torch>=2.2.2' -# To resolve vulnerability issue +# To resolve GHSA-jh2j-j4j9-crg3 vulnerability issue RUN pip install 'opencv-python-headless==4.8.1.78' +# clean conda and pip caches +RUN conda run -p $CONDA_PREFIX pip cache purge && \ + conda clean -a -y + +RUN rm -rf ~/.cache/pip + RUN pip freeze # This is needed for mpi to locate libpython -ENV LD_LIBRARY_PATH $AZUREML_CONDA_ENVIRONMENT_PATH/lib:$LD_LIBRARY_PATH +ENV LD_LIBRARY_PATH $CONDA_PREFIX/lib:$LD_LIBRARY_PATH diff --git a/assets/responsibleai/environments/responsibleai-vision/context/conda_dependencies.yaml b/assets/responsibleai/environments/responsibleai-vision/context/conda_dependencies.yaml index 812da53cec..8c6dac250d 100644 --- a/assets/responsibleai/environments/responsibleai-vision/context/conda_dependencies.yaml +++ b/assets/responsibleai/environments/responsibleai-vision/context/conda_dependencies.yaml @@ -1,7 +1,6 @@ name: responsibleai-vision channels: - conda-forge - - defaults - anaconda - pytorch dependencies: diff --git a/assets/responsibleai/environments/responsibleai-vision/tests/requirements.txt b/assets/responsibleai/environments/responsibleai-vision/tests/requirements.txt index cf979ad62f..a808fc9af1 100644 --- a/assets/responsibleai/environments/responsibleai-vision/tests/requirements.txt +++ b/assets/responsibleai/environments/responsibleai-vision/tests/requirements.txt @@ -1,3 +1,3 @@ -azure-ai-ml==1.22.4 -azure.identity==1.19.0 +azure-ai-ml==1.25.0 +azure-identity==1.20.0 azureml-core>=1.58.0 \ No newline at end of file diff --git a/scripts/test/create_asset_test_matrix.py b/scripts/test/create_asset_test_matrix.py index 19e39ec301..9c366cba9a 100644 --- a/scripts/test/create_asset_test_matrix.py +++ b/scripts/test/create_asset_test_matrix.py @@ -53,6 +53,7 @@ def create_test_matrix(input_dirs: List[Path], for counter_name in COUNTERS: logger.set_output(counter_name, counters[counter_name]) logger.set_output(MATRIX, json.dumps({'asset_config_path': asset_config_files})) + logger.set_output(MATRIX, json.dumps({'asset_config_name': asset_config_files.replace("/", "-")})) if __name__ == '__main__':