From 3fae405b82643875651195ab74f4722ca7594c59 Mon Sep 17 00:00:00 2001 From: Michael Norris Date: Fri, 17 Jan 2025 15:25:09 -0800 Subject: [PATCH] Migration off defaults to conda-forge channel (#4126) Summary: Good resource on overriding channels to make sure we aren't using `defaults`:https://stackoverflow.com/questions/67695893/how-do-i-completely-purge-and-disable-the-default-channel-in-anaconda-and-switch Explanation of changes: - - changed to miniforge from miniconda: this ensures we only pull in from conda-defaults when creating the environment - architecture change: ARM64 and aarch64 are the same thing. But there is no miniforge package for ARM64, so we need to make it check for aarch64 instead. - mkl 2022.2.1 change: conda-forge and defaults have completely different dependencies. Defaults required intel-openmp, but now on conda-forge, mkl 2023.1 or higher requires llvm-openmp >=14.0.6, but this is incompatible with the pytorch build <2.5 which requires llvm-openmp<14.0. We would need to upgrade Python to 3.12 first, upgrade Pytorch build, then upgrade this mkl. (The meta.yaml changes are the ones that narrow it to 2022.2.1 during `conda build faiss`.) So, this has just been changed to 2022.2.1. - mkl now requires _openmp_mutex of type "llvm" instead of "gnu": prior non-cuVS builds all used gnu, because intel-openmp from anaconda defaults channel does not require llvm-openmp. Now we need to remove the gnu one which is automatically pulled in during miniconda setup, and only keep the llvm version of _openmp_mutex. - liblief: The above changes tried to pull in liblief 0.15. This results in an error like `AttributeError: module 'lief._lief.ELF' has no attribute 'ELF_CLASS'`. When I checked passing PR builds on defaults, they use lief 0.12, so I pinned that one. Differential Revision: D68043874 --- .github/actions/build_cmake/action.yml | 17 ++- .github/actions/build_conda/action.yml | 20 +++- .github/workflows/build-pull-request.yml | 127 +++++++++++++++++++++++ conda/conda_build_config.yaml | 1 - conda/faiss-gpu/meta.yaml | 9 +- conda/faiss/meta.yaml | 21 ++-- 6 files changed, 179 insertions(+), 16 deletions(-) diff --git a/.github/actions/build_cmake/action.yml b/.github/actions/build_cmake/action.yml index fa20974af5..a5f9372aec 100644 --- a/.github/actions/build_cmake/action.yml +++ b/.github/actions/build_cmake/action.yml @@ -23,12 +23,19 @@ runs: uses: conda-incubator/setup-miniconda@v3 with: python-version: '3.11' - miniconda-version: latest + miniforge-version: latest # ensures conda-forge channel is used. + channels: conda-forge + conda-remove-defaults: 'true' + # Set to aarch64 if we're on arm64 because there's no miniforge ARM64 package, just aarch64. + # They are the same thing, just named differently. + architecture: ${{ runner.arch == 'ARM64' && 'aarch64' || runner.arch }} - name: Configure build environment shell: bash run: | # initialize Conda conda config --set solver libmamba + # Ensure starting packages are from conda-forge. + conda list --show-channel-urls conda update -y -q conda echo "$CONDA/bin" >> $GITHUB_PATH @@ -43,7 +50,7 @@ runs: if [ "${{ runner.arch }}" = "X64" ]; then # TODO: merge this with ARM64 conda install -y -q -c conda-forge gxx_linux-64=14.2 sysroot_linux-64=2.17 - conda install -y -q mkl=2023 mkl-devel=2023 + conda install -y -q mkl=2022.2.1 mkl-devel=2022.2.1 fi # no CUDA needed for ROCm so skip this @@ -56,6 +63,7 @@ runs: elif [ "${{ inputs.cuvs }}" = "ON" ]; then conda install -y -q libcuvs=24.12 'cuda-version>=12.0,<=12.5' cuda-toolkit=12.4.1 gxx_linux-64=12.4 -c rapidsai -c conda-forge fi + # install test packages if [ "${{ inputs.rocm }}" = "ON" ]; then : # skip torch install via conda, we need to install via pip to get @@ -174,3 +182,8 @@ runs: with: name: test-results-arch=${{ runner.arch }}-opt=${{ inputs.opt_level }}-gpu=${{ inputs.gpu }}-cuvs=${{ inputs.cuvs }}-rocm=${{ inputs.rocm }} path: test-results + - name: Check installed packages channel + shell: bash + run: | + # Shows that all installed packages are from conda-forge. + conda list --show-channel-urls diff --git a/.github/actions/build_conda/action.yml b/.github/actions/build_conda/action.yml index ff860007b2..d7e02e606e 100644 --- a/.github/actions/build_conda/action.yml +++ b/.github/actions/build_conda/action.yml @@ -26,20 +26,25 @@ runs: else echo "shell=pwsh" >> "$GITHUB_OUTPUT" fi + echo "${{ runner.arch }}" - name: Setup miniconda uses: conda-incubator/setup-miniconda@v3 with: python-version: '3.11' - miniconda-version: latest + miniforge-version: latest # ensures conda-forge channel is used. + channels: conda-forge + conda-remove-defaults: 'true' + # Set to aarch64 if we're on arm64 because there's no miniforge ARM64 package, just aarch64. + # They are the same thing, just named differently. + architecture: ${{ (runner.arch == 'ARM64' && runner.os != 'macOS') && 'aarch64' || runner.arch }} - name: Install conda build tools shell: ${{ steps.choose_shell.outputs.shell }} run: | + # Ensure starting packages are from conda-forge. + conda list --show-channel-urls conda install -y -q "conda!=24.11.0" conda install -y -q "conda-build!=24.11.0" - - name: Fix CI failure - shell: ${{ steps.choose_shell.outputs.shell }} - if: runner.os != 'Windows' - run: conda remove conda-anaconda-telemetry + conda list --show-channel-urls - name: Enable anaconda uploads if: inputs.label != '' shell: ${{ steps.choose_shell.outputs.shell }} @@ -94,3 +99,8 @@ runs: run: | conda build faiss-gpu-cuvs --variants '{ "cudatoolkit": "${{ inputs.cuda }}" }' \ --user pytorch --label ${{ inputs.label }} -c pytorch -c rapidsai -c rapidsai-nightly -c conda-forge -c nvidia + - name: Check installed packages channel + shell: ${{ steps.choose_shell.outputs.shell }} + run: | + # Shows that all installed packages are from conda-forge. + conda list --show-channel-urls diff --git a/.github/workflows/build-pull-request.yml b/.github/workflows/build-pull-request.yml index bc0d2d625a..ee8f3cae31 100644 --- a/.github/workflows/build-pull-request.yml +++ b/.github/workflows/build-pull-request.yml @@ -167,3 +167,130 @@ jobs: fetch-tags: true - name: Build and Package (conda) uses: ./.github/actions/build_conda + linux-x86_64-nightly: + name: Linux x86_64 nightlies + runs-on: 4-core-ubuntu + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + - uses: ./.github/actions/build_conda + env: + ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} + with: + label: nightly + linux-x86_64-GPU-CUDA-11-4-4-nightly: + name: Linux x86_64 GPU nightlies (CUDA 11.4.4) + runs-on: 4-core-ubuntu-gpu-t4 + env: + CUDA_ARCHS: "60-real;61-real;62-real;70-real;72-real;75-real;80;86-real" + FAISS_FLATTEN_CONDA_INCLUDES: "1" + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + - uses: ./.github/actions/build_conda + env: + ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} + with: + label: nightly + cuda: "11.4.4" + linux-x86_64-GPU-CUVS-CUDA11-8-0-nightly: + name: Linux x86_64 GPU w/ cuVS nightlies (CUDA 11.8.0) + runs-on: 4-core-ubuntu-gpu-t4 + env: + CUDA_ARCHS: "70-real;72-real;75-real;80;86-real" + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + - uses: ./.github/actions/build_conda + env: + ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} + with: + label: nightly + cuvs: "ON" + cuda: "11.8.0" + linux-x86_64-GPU-CUDA-12-1-1-nightly: + name: Linux x86_64 GPU nightlies (CUDA 12.1.1) + runs-on: 4-core-ubuntu-gpu-t4 + env: + CUDA_ARCHS: "70-real;72-real;75-real;80;86-real" + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + - uses: ./.github/actions/build_conda + env: + ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} + with: + label: nightly + cuda: "12.1.1" + linux-x86_64-GPU-CUVS-CUDA12-4-0-nightly: + name: Linux x86_64 GPU w/ cuVS nightlies (CUDA 12.4.0) + runs-on: 4-core-ubuntu-gpu-t4 + env: + CUDA_ARCHS: "70-real;72-real;75-real;80;86-real" + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + - uses: ./.github/actions/build_conda + env: + ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} + with: + label: nightly + cuvs: "ON" + cuda: "12.4.0" + windows-x86_64-nightly: + name: Windows x86_64 nightlies + runs-on: windows-2019 + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + - uses: ./.github/actions/build_conda + env: + ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} + with: + label: nightly + osx-arm64-nightly: + name: OSX arm64 nightlies + runs-on: macos-14 + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + - uses: ./.github/actions/build_conda + env: + ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} + with: + label: nightly + linux-arm64-nightly: + name: Linux arm64 nightlies + runs-on: 2-core-ubuntu-arm + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + - uses: ./.github/actions/build_conda + env: + ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} + with: + label: nightly diff --git a/conda/conda_build_config.yaml b/conda/conda_build_config.yaml index 4df05146df..77f0eec0a2 100644 --- a/conda/conda_build_config.yaml +++ b/conda/conda_build_config.yaml @@ -2,4 +2,3 @@ python: - 3.9 - 3.10 - 3.11 - - 3.12 # [not aarch64] diff --git a/conda/faiss-gpu/meta.yaml b/conda/faiss-gpu/meta.yaml index 651d42fefa..20c5fb6d2d 100644 --- a/conda/faiss-gpu/meta.yaml +++ b/conda/faiss-gpu/meta.yaml @@ -50,9 +50,11 @@ outputs: - sysroot_linux-64 =2.17 # [linux64] - llvm-openmp # [osx] - cmake >=3.24.0 - - make =4.2 # [not win] + - make =4.2 # [not win and not (osx and arm64)] + - make =4.4 # [osx and arm64] - mkl-devel =2023 # [x86_64] - cuda-toolkit {{ cudatoolkit }} + - libcusolver >=11.4 host: - mkl =2023 # [x86_64] - openblas =0.3 # [not x86_64] @@ -83,11 +85,14 @@ outputs: - sysroot_linux-64 =2.17 # [linux64] - swig =4.0 - cmake >=3.24.0 - - make =4.2 # [not win] + - make =4.2 # [not win and not (osx and arm64)] + - make =4.4 # [osx and arm64] + - _openmp_mutex =4.5=2_kmp_llvm # [x86_64 and not win] - cuda-toolkit {{ cudatoolkit }} host: - python {{ python }} - numpy >=1.19,<2 + - _openmp_mutex =4.5=2_kmp_llvm # [x86_64 and not win] - {{ pin_subpackage('libfaiss', exact=True) }} run: - python {{ python }} diff --git a/conda/faiss/meta.yaml b/conda/faiss/meta.yaml index fe7612c23b..c1f4d09a14 100644 --- a/conda/faiss/meta.yaml +++ b/conda/faiss/meta.yaml @@ -38,15 +38,18 @@ outputs: build: - {{ compiler('cxx') }} - sysroot_linux-64 =2.17 # [linux64] - - llvm-openmp # [osx] + - llvm-openmp # [osx or linux64] - cmake >=3.24.0 - - make =4.2 # [not win] - - mkl-devel =2023 # [x86_64] + - make =4.2 # [not win and not (osx and arm64)] + - make =4.4 # [osx and arm64] + - mkl-devel =2022.2.1 # [x86_64] + - liblief =0.12.3 # [not win] host: - - mkl =2023 # [x86_64] + - mkl =2022.2.1 # [x86_64] - openblas =0.3 # [not x86_64] + - liblief =0.12.3 # [not win] run: - - mkl =2023 # [x86_64] + - mkl =2022.2.1 # [x86_64] - openblas =0.3 # [not x86_64] test: requires: @@ -70,16 +73,22 @@ outputs: - sysroot_linux-64 =2.17 # [linux64] - swig =4.0 - cmake >=3.24.0 - - make =4.2 # [not win] + - make =4.2 # [not win and not (osx and arm64)] + - make =4.4 # [osx and arm64] + - _openmp_mutex =4.5=2_kmp_llvm # [x86_64 and not win] + - mkl =2022.2.1 # [win] host: - python {{ python }} - numpy >=1.19,<2 - {{ pin_subpackage('libfaiss', exact=True) }} + - _openmp_mutex =4.5=2_kmp_llvm # [x86_64 and not win] + - mkl =2022.2.1 # [win] run: - python {{ python }} - numpy >=1.19,<2 - packaging - {{ pin_subpackage('libfaiss', exact=True) }} + - mkl =2022.2.1 # [win] test: requires: - numpy >=1.19,<2