Migration off defaults to conda-forge channel (#4126)

Summary: Good resource on overriding channels to make sure we aren't using `defaults`:https://stackoverflow.com/questions/67695893/how-do-i-completely-purge-and-disable-the-default-channel-in-anaconda-and-switch Explanation of changes: - - changed to miniforge from miniconda: this ensures we only pull in from conda-defaults when creating the environment - architecture change: ARM64 and aarch64 are the same thing. But there is no miniforge package for ARM64, so we need to make it check for aarch64 instead. - mkl 2022.2.1 change: conda-forge and defaults have completely different dependencies. Defaults required intel-openmp, but now on conda-forge, mkl 2023.1 or higher requires llvm-openmp >=14.0.6, but this is incompatible with the pytorch build <2.5 which requires llvm-openmp<14.0. We would need to upgrade Python to 3.12 first, upgrade Pytorch build, then upgrade this mkl. (The meta.yaml changes are the ones that narrow it to 2022.2.1 during `conda build faiss`.) So, this has just been changed to 2022.2.1. - mkl now requires _openmp_mutex of type "llvm" instead of "gnu": prior non-cuVS builds all used gnu, because intel-openmp from anaconda defaults channel does not require llvm-openmp. Now we need to remove the gnu one which is automatically pulled in during miniconda setup, and only keep the llvm version of _openmp_mutex. - liblief: The above changes tried to pull in liblief 0.15. This results in an error like `AttributeError: module 'lief._lief.ELF' has no attribute 'ELF_CLASS'`. When I checked passing PR builds on defaults, they use lief 0.12, so I pinned that one. Differential Revision: D68043874
facebookresearch · Jan 18, 2025 · 30d83d9 · 30d83d9
1 parent 4c315a9
commit 30d83d9
Show file tree

Hide file tree

Showing 6 changed files with 179 additions and 16 deletions.
diff --git a/.github/actions/build_cmake/action.yml b/.github/actions/build_cmake/action.yml
@@ -23,12 +23,19 @@ runs:
       uses: conda-incubator/setup-miniconda@v3
       with:
         python-version: '3.11'
-        miniconda-version: latest
+        miniforge-version: latest # ensures conda-forge channel is used.
+        channels: conda-forge
+        conda-remove-defaults: 'true'
+        # Set to aarch64 if we're on arm64 because there's no miniforge ARM64 package, just aarch64.
+        # They are the same thing, just named differently.
+        architecture: ${{ runner.arch  == 'ARM64' && 'aarch64' || runner.arch }}
     - name: Configure build environment
       shell: bash
       run: |
         # initialize Conda
         conda config --set solver libmamba
+        # Ensure starting packages are from conda-forge.
+        conda list --show-channel-urls
         conda update -y -q conda
         echo "$CONDA/bin" >> $GITHUB_PATH
 
@@ -43,7 +50,7 @@ runs:
         if [ "${{ runner.arch }}" = "X64" ]; then
           # TODO: merge this with ARM64
           conda install -y -q -c conda-forge gxx_linux-64=14.2 sysroot_linux-64=2.17
-          conda install -y -q mkl=2023 mkl-devel=2023
+          conda install -y -q mkl=2022.2.1 mkl-devel=2022.2.1
         fi
 
         # no CUDA needed for ROCm so skip this
@@ -56,6 +63,7 @@ runs:
         elif [ "${{ inputs.cuvs }}" = "ON" ]; then
           conda install -y -q libcuvs=24.12 'cuda-version>=12.0,<=12.5' cuda-toolkit=12.4.1 gxx_linux-64=12.4 -c rapidsai -c conda-forge
         fi
+
         # install test packages
         if [ "${{ inputs.rocm }}" = "ON" ]; then
           : # skip torch install via conda, we need to install via pip to get
@@ -174,3 +182,8 @@ runs:
       with:
         name: test-results-arch=${{ runner.arch }}-opt=${{ inputs.opt_level }}-gpu=${{ inputs.gpu }}-cuvs=${{ inputs.cuvs }}-rocm=${{ inputs.rocm }}
         path: test-results
+    - name: Check installed packages channel
+      shell: bash
+      run: |
+        # Shows that all installed packages are from conda-forge.
+        conda list --show-channel-urls
diff --git a/.github/actions/build_conda/action.yml b/.github/actions/build_conda/action.yml
@@ -26,20 +26,25 @@ runs:
         else
           echo "shell=pwsh" >> "$GITHUB_OUTPUT"
         fi
+        echo "${{ runner.arch }}"
     - name: Setup miniconda
       uses: conda-incubator/setup-miniconda@v3
       with:
         python-version: '3.11'
-        miniconda-version:  latest
+        miniforge-version: latest # ensures conda-forge channel is used.
+        channels: conda-forge
+        conda-remove-defaults: 'true'
+        # Set to aarch64 if we're on arm64 because there's no miniforge ARM64 package, just aarch64.
+        # They are the same thing, just named differently.
+        architecture: ${{ (runner.arch  == 'ARM64' && runner.os != 'macOS') && 'aarch64' || runner.arch }}
     - name: Install conda build tools
       shell: ${{ steps.choose_shell.outputs.shell }}
       run: |
+        # Ensure starting packages are from conda-forge.
+        conda list --show-channel-urls
         conda install -y -q "conda!=24.11.0"
         conda install -y -q "conda-build!=24.11.0"
-    - name: Fix CI failure
-      shell: ${{ steps.choose_shell.outputs.shell }}
-      if: runner.os != 'Windows'
-      run: conda remove conda-anaconda-telemetry
+        conda list --show-channel-urls
     - name: Enable anaconda uploads
       if: inputs.label != ''
       shell: ${{ steps.choose_shell.outputs.shell }}
@@ -94,3 +99,8 @@ runs:
       run: |
         conda build faiss-gpu-cuvs --variants '{ "cudatoolkit": "${{ inputs.cuda }}" }' \
             --user pytorch --label ${{ inputs.label }} -c pytorch -c rapidsai -c rapidsai-nightly -c conda-forge -c nvidia
+    - name: Check installed packages channel
+      shell: ${{ steps.choose_shell.outputs.shell }}
+      run: |
+        # Shows that all installed packages are from conda-forge.
+        conda list --show-channel-urls
diff --git a/.github/workflows/build-pull-request.yml b/.github/workflows/build-pull-request.yml
@@ -167,3 +167,130 @@ jobs:
           fetch-tags: true
       - name: Build and Package (conda)
         uses: ./.github/actions/build_conda
+  linux-x86_64-nightly:
+    name: Linux x86_64 nightlies
+    runs-on: 4-core-ubuntu
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          fetch-tags: true
+      - uses: ./.github/actions/build_conda
+        env:
+          ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }}
+        with:
+          label: nightly
+  linux-x86_64-GPU-CUDA-11-4-4-nightly:
+    name: Linux x86_64 GPU nightlies (CUDA 11.4.4)
+    runs-on: 4-core-ubuntu-gpu-t4
+    env:
+      CUDA_ARCHS: "60-real;61-real;62-real;70-real;72-real;75-real;80;86-real"
+      FAISS_FLATTEN_CONDA_INCLUDES: "1"
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          fetch-tags: true
+      - uses: ./.github/actions/build_conda
+        env:
+          ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }}
+        with:
+          label: nightly
+          cuda: "11.4.4"
+  linux-x86_64-GPU-CUVS-CUDA11-8-0-nightly:
+    name: Linux x86_64 GPU w/ cuVS nightlies (CUDA 11.8.0)
+    runs-on: 4-core-ubuntu-gpu-t4
+    env:
+      CUDA_ARCHS: "70-real;72-real;75-real;80;86-real"
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          fetch-tags: true
+      - uses: ./.github/actions/build_conda
+        env:
+          ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }}
+        with:
+          label: nightly
+          cuvs: "ON"
+          cuda: "11.8.0"
+  linux-x86_64-GPU-CUDA-12-1-1-nightly:
+    name: Linux x86_64 GPU nightlies (CUDA 12.1.1)
+    runs-on: 4-core-ubuntu-gpu-t4
+    env:
+      CUDA_ARCHS: "70-real;72-real;75-real;80;86-real"
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          fetch-tags: true
+      - uses: ./.github/actions/build_conda
+        env:
+          ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }}
+        with:
+          label: nightly
+          cuda: "12.1.1"
+  linux-x86_64-GPU-CUVS-CUDA12-4-0-nightly:
+    name: Linux x86_64 GPU w/ cuVS nightlies (CUDA 12.4.0)
+    runs-on: 4-core-ubuntu-gpu-t4
+    env:
+      CUDA_ARCHS: "70-real;72-real;75-real;80;86-real"
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          fetch-tags: true
+      - uses: ./.github/actions/build_conda
+        env:
+          ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }}
+        with:
+          label: nightly
+          cuvs: "ON"
+          cuda: "12.4.0"
+  windows-x86_64-nightly:
+    name: Windows x86_64 nightlies
+    runs-on: windows-2019
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          fetch-tags: true
+      - uses: ./.github/actions/build_conda
+        env:
+          ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }}
+        with:
+          label: nightly
+  osx-arm64-nightly:
+    name: OSX arm64 nightlies
+    runs-on: macos-14
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          fetch-tags: true
+      - uses: ./.github/actions/build_conda
+        env:
+          ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }}
+        with:
+          label: nightly
+  linux-arm64-nightly:
+    name: Linux arm64 nightlies
+    runs-on: 2-core-ubuntu-arm
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          fetch-tags: true
+      - uses: ./.github/actions/build_conda
+        env:
+          ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }}
+        with:
+          label: nightly
diff --git a/conda/conda_build_config.yaml b/conda/conda_build_config.yaml
@@ -2,4 +2,3 @@ python:
   - 3.9
   - 3.10
   - 3.11
-  - 3.12  # [not aarch64]
diff --git a/conda/faiss-gpu/meta.yaml b/conda/faiss-gpu/meta.yaml
@@ -50,9 +50,11 @@ outputs:
         - sysroot_linux-64 =2.17 # [linux64]
         - llvm-openmp  # [osx]
         - cmake >=3.24.0
-        - make =4.2 # [not win]
+        - make =4.2 # [not win and not (osx and arm64)]
+        - make =4.4 # [osx and arm64]
         - mkl-devel =2023  # [x86_64]
         - cuda-toolkit {{ cudatoolkit }}
+        - gcc_linux-64 =11.2  # [{{ cudatoolkit }} == '11.4.4']
       host:
         - mkl =2023  # [x86_64]
         - openblas =0.3 # [not x86_64]
@@ -83,11 +85,14 @@ outputs:
         - sysroot_linux-64 =2.17 # [linux64]
         - swig =4.0
         - cmake >=3.24.0
-        - make =4.2 # [not win]
+        - make =4.2 # [not win and not (osx and arm64)]
+        - make =4.4 # [osx and arm64]
+        - _openmp_mutex =4.5=2_kmp_llvm  # [x86_64 and not win]
         - cuda-toolkit {{ cudatoolkit }}
       host:
         - python {{ python }}
         - numpy >=1.19,<2
+        - _openmp_mutex =4.5=2_kmp_llvm  # [x86_64 and not win]
         - {{ pin_subpackage('libfaiss', exact=True) }}
       run:
         - python {{ python }}

diff --git a/conda/faiss/meta.yaml b/conda/faiss/meta.yaml
@@ -38,15 +38,18 @@ outputs:
       build:
         - {{ compiler('cxx') }}
         - sysroot_linux-64 =2.17 # [linux64]
-        - llvm-openmp  # [osx]
+        - llvm-openmp  # [osx or linux64]
         - cmake >=3.24.0
-        - make =4.2 # [not win]
-        - mkl-devel =2023  # [x86_64]
+        - make =4.2 # [not win and not (osx and arm64)]
+        - make =4.4 # [osx and arm64]
+        - mkl-devel =2022.2.1  # [x86_64]
+        - liblief =0.12.3  # [not win]
       host:
-        - mkl =2023  # [x86_64]
+        - mkl =2022.2.1  # [x86_64]
         - openblas =0.3 # [not x86_64]
+        - liblief =0.12.3  # [not win]
       run:
-        - mkl =2023  # [x86_64]
+        - mkl =2022.2.1  # [x86_64]
         - openblas =0.3 # [not x86_64]
     test:
       requires:
@@ -70,16 +73,22 @@ outputs:
         - sysroot_linux-64 =2.17 # [linux64]
         - swig =4.0
         - cmake >=3.24.0
-        - make =4.2 # [not win]
+        - make =4.2 # [not win and not (osx and arm64)]
+        - make =4.4 # [osx and arm64]
+        - _openmp_mutex =4.5=2_kmp_llvm  # [x86_64 and not win]
+        - mkl =2022.2.1  # [win]
       host:
         - python {{ python }}
         - numpy >=1.19,<2
         - {{ pin_subpackage('libfaiss', exact=True) }}
+        - _openmp_mutex =4.5=2_kmp_llvm  # [x86_64 and not win]
+        - mkl =2022.2.1  # [win]
       run:
         - python {{ python }}
         - numpy >=1.19,<2
         - packaging
         - {{ pin_subpackage('libfaiss', exact=True) }}
+        - mkl =2022.2.1  # [win]
     test:
       requires:
         - numpy >=1.19,<2
-Original file line number
+Diff line change
@@ Expand Up / @@ -2,4 +2,3 @@ python: @@
       - 3.9
       - 3.10
       - 3.11
-      - 3.12  # [not aarch64]