Skip to content

Commit

Permalink
Merge branch 'branch-25.04' into replace_cub_transform_iterator
Browse files Browse the repository at this point in the history
  • Loading branch information
cjnolet authored Feb 18, 2025
2 parents 23cf2aa + d99d285 commit 07948cb
Show file tree
Hide file tree
Showing 208 changed files with 8,022 additions and 13,294 deletions.
13 changes: 13 additions & 0 deletions .github/workflows/build.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,19 @@ jobs:
node_type: "gpu-l4-latest-1"
run_script: "ci/build_rust.sh"
sha: ${{ inputs.sha }}
go-build:
needs: cpp-build
secrets: inherit
uses: rapidsai/shared-workflows/.github/workflows/[email protected]
with:
build_type: ${{ inputs.build_type || 'branch' }}
branch: ${{ inputs.branch }}
arch: "amd64"
date: ${{ inputs.date }}
container_image: "rapidsai/ci-conda:latest"
node_type: "gpu-v100-latest-1"
run_script: "ci/build_go.sh"
sha: ${{ inputs.sha }}
python-build:
needs: [cpp-build]
secrets: inherit
Expand Down
39 changes: 39 additions & 0 deletions .github/workflows/pr.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -22,16 +22,28 @@ jobs:
- conda-python-tests
- docs-build
- rust-build
- go-build
- wheel-build-libcuvs
- wheel-build-cuvs
- wheel-tests-cuvs
- devcontainer
- telemetry-setup
secrets: inherit
uses: rapidsai/shared-workflows/.github/workflows/[email protected]
if: always()
with:
needs: ${{ toJSON(needs) }}
telemetry-setup:
continue-on-error: true
runs-on: ubuntu-latest
env:
OTEL_SERVICE_NAME: 'pr-cuvs'
steps:
- name: Telemetry setup
if: ${{ vars.TELEMETRY_ENABLED == 'true' }}
uses: rapidsai/shared-actions/telemetry-dispatch-stash-base-env-vars@main
check-nightly-ci:
needs: telemetry-setup
# Switch to ubuntu-latest once it defaults to a version of Ubuntu that
# provides at least Python 3.11 (see
# https://docs.python.org/3/library/datetime.html#datetime.date.fromisoformat)
Expand All @@ -44,6 +56,7 @@ jobs:
with:
repo: cuvs
changed-files:
needs: telemetry-setup
secrets: inherit
uses: rapidsai/shared-workflows/.github/workflows/[email protected]
with:
Expand All @@ -58,13 +71,15 @@ jobs:
- '!notebooks/**'
- '!python/**'
- '!rust/**'
- '!go/**'
- '!thirdparty/LICENSES/**'
test_notebooks:
- '**'
- '!.devcontainer/**'
- '!.pre-commit-config.yaml'
- '!README.md'
- '!rust/**'
- '!go/**'
- '!thirdparty/LICENSES/**'
test_python:
- '**'
Expand All @@ -75,12 +90,15 @@ jobs:
- '!img/**'
- '!notebooks/**'
- '!rust/**'
- '!go/**'
- '!thirdparty/LICENSES/**'
checks:
needs: telemetry-setup
secrets: inherit
uses: rapidsai/shared-workflows/.github/workflows/[email protected]
with:
enable_check_generated_files: false
ignored_pr_jobs: "telemetry-summarize"
conda-cpp-build:
needs: checks
secrets: inherit
Expand Down Expand Up @@ -136,6 +154,16 @@ jobs:
arch: "amd64"
container_image: "rapidsai/ci-conda:latest"
run_script: "ci/build_rust.sh"
go-build:
needs: conda-cpp-build
secrets: inherit
uses: rapidsai/shared-workflows/.github/workflows/[email protected]
with:
build_type: pull-request
node_type: "gpu-v100-latest-1"
arch: "amd64"
container_image: "rapidsai/ci-conda:latest"
run_script: "ci/build_go.sh"
wheel-build-libcuvs:
needs: checks
secrets: inherit
Expand All @@ -162,6 +190,7 @@ jobs:
script: ci/test_wheel_cuvs.sh
devcontainer:
secrets: inherit
needs: telemetry-setup
uses: rapidsai/shared-workflows/.github/workflows/[email protected]
with:
arch: '["amd64"]'
Expand All @@ -170,3 +199,13 @@ jobs:
sccache -z;
build-all --verbose;
sccache -s;
telemetry-summarize:
# This job must use a self-hosted runner to record telemetry traces.
runs-on: linux-amd64-cpu4
needs: pr-builder
if: ${{ vars.TELEMETRY_ENABLED == 'true' && !cancelled() }}
continue-on-error: true
steps:
- name: Telemetry summarize
uses: rapidsai/shared-actions/telemetry-dispatch-summarize@main
104 changes: 104 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,107 @@
# cuvs 25.02.00 (13 Feb 2025)

## 🚨 Breaking Changes

- update pip devcontainers to UCX 1.18, small update-version.sh fixes ([#604](https://github.com/rapidsai/cuvs/pull/604)) [@jameslamb](https://github.com/jameslamb)
- Improve the performance of CAGRA new vector addition with the default params ([#569](https://github.com/rapidsai/cuvs/pull/569)) [@enp1s0](https://github.com/enp1s0)
- Update for raft logger changes ([#540](https://github.com/rapidsai/cuvs/pull/540)) [@vyasr](https://github.com/vyasr)

## 🐛 Bug Fixes

- Fix ann-bench dataset blob integer overflow leading to incorrect data copy beyond 4B elems ([#671](https://github.com/rapidsai/cuvs/pull/671)) [@achirkin](https://github.com/achirkin)
- Fix ann-bench deadlocking on HNSW destruction due to task locks ([#667](https://github.com/rapidsai/cuvs/pull/667)) [@achirkin](https://github.com/achirkin)
- cuvs-bench Fixes ([#654](https://github.com/rapidsai/cuvs/pull/654)) [@tarang-jain](https://github.com/tarang-jain)
- Fix std::lock_guard use for gcc 14 support ([#639](https://github.com/rapidsai/cuvs/pull/639)) [@enp1s0](https://github.com/enp1s0)
- Fix indexing bug when using parallelism to build CPU hierarchy in HNSW ([#620](https://github.com/rapidsai/cuvs/pull/620)) [@divyegala](https://github.com/divyegala)
- add runtime dependency on libcuvs in cuvs wheels ([#615](https://github.com/rapidsai/cuvs/pull/615)) [@jameslamb](https://github.com/jameslamb)
- Temporarily skip CUDA 11 wheel CI ([#599](https://github.com/rapidsai/cuvs/pull/599)) [@bdice](https://github.com/bdice)
- [Fix] l2_exp random fail in half-float32 mixed precision on self-neighboring ([#596](https://github.com/rapidsai/cuvs/pull/596)) [@rhdong](https://github.com/rhdong)
- Add CAGRA InnerProduct test and fix a bug ([#595](https://github.com/rapidsai/cuvs/pull/595)) [@enp1s0](https://github.com/enp1s0)
- fix cuvs_bench.run --groups options ([#592](https://github.com/rapidsai/cuvs/pull/592)) [@jiangyinzuo](https://github.com/jiangyinzuo)
- Fix cagra_hnsw serialization when dataset is not part of index ([#591](https://github.com/rapidsai/cuvs/pull/591)) [@tfeher](https://github.com/tfeher)
- fix create_pointset for throughput mode ([#589](https://github.com/rapidsai/cuvs/pull/589)) [@jiangyinzuo](https://github.com/jiangyinzuo)
- Fix the use of constexpr in the dynamic batching header ([#582](https://github.com/rapidsai/cuvs/pull/582)) [@achirkin](https://github.com/achirkin)
- Reduce the recall threshold for IVF-PQ low-precision LUT inner product tests ([#573](https://github.com/rapidsai/cuvs/pull/573)) [@achirkin](https://github.com/achirkin)
- Small fixes to docs and pairwise distances ([#570](https://github.com/rapidsai/cuvs/pull/570)) [@cjnolet](https://github.com/cjnolet)
- [BUG] Fix CAGRA graph optimization bug ([#565](https://github.com/rapidsai/cuvs/pull/565)) [@enp1s0](https://github.com/enp1s0)
- Fix broken link to python doc ([#564](https://github.com/rapidsai/cuvs/pull/564)) [@lowener](https://github.com/lowener)
- Fix cagra::extend error message ([#532](https://github.com/rapidsai/cuvs/pull/532)) [@enp1s0](https://github.com/enp1s0)
- Fix Grace-specific issues in CAGRA ([#527](https://github.com/rapidsai/cuvs/pull/527)) [@achirkin](https://github.com/achirkin)

## 📖 Documentation

- add docs for nn_descent ([#668](https://github.com/rapidsai/cuvs/pull/668)) [@Intron7](https://github.com/Intron7)
- Fixing small typo in cuvs bench docs ([#586](https://github.com/rapidsai/cuvs/pull/586)) [@cjnolet](https://github.com/cjnolet)
- Fix typos in README ([#543](https://github.com/rapidsai/cuvs/pull/543)) [@nvanbenschoten](https://github.com/nvanbenschoten)
- Use nvidia-sphinx-theme for docs ([#528](https://github.com/rapidsai/cuvs/pull/528)) [@benfred](https://github.com/benfred)

## 🚀 New Features

- Add deep-100M to datasets.yaml for cuvs-bench ([#670](https://github.com/rapidsai/cuvs/pull/670)) [@tarang-jain](https://github.com/tarang-jain)
- Expose configuration singleton as a global context for ann-bench algos ([#647](https://github.com/rapidsai/cuvs/pull/647)) [@achirkin](https://github.com/achirkin)
- ANN_BENCH enhanced dataset support ([#624](https://github.com/rapidsai/cuvs/pull/624)) [@achirkin](https://github.com/achirkin)
- [Feat] Add Support for Index `merge` in CAGRA ([#618](https://github.com/rapidsai/cuvs/pull/618)) [@rhdong](https://github.com/rhdong)
- HNSW GPU hierarchy ([#616](https://github.com/rapidsai/cuvs/pull/616)) [@divyegala](https://github.com/divyegala)
- CAGRA binary Hamming distance support ([#610](https://github.com/rapidsai/cuvs/pull/610)) [@enp1s0](https://github.com/enp1s0)
- Add cuda 12.8 support ([#605](https://github.com/rapidsai/cuvs/pull/605)) [@robertmaynard](https://github.com/robertmaynard)
- Add support for refinement with `uint32_t` index type ([#563](https://github.com/rapidsai/cuvs/pull/563)) [@lowener](https://github.com/lowener)
- [Feat] Support `bitset` filter for Brute Force ([#560](https://github.com/rapidsai/cuvs/pull/560)) [@rhdong](https://github.com/rhdong)
- Remove upper bounds on cuda-python to allow 12.6.2 and 11.8.5 ([#508](https://github.com/rapidsai/cuvs/pull/508)) [@bdice](https://github.com/bdice)

## 🛠️ Improvements

- Add filtering to python for ivf_flat ([#664](https://github.com/rapidsai/cuvs/pull/664)) [@benfred](https://github.com/benfred)
- Expose binary quantizer to C and Python ([#660](https://github.com/rapidsai/cuvs/pull/660)) [@benfred](https://github.com/benfred)
- Add telemetry ([#652](https://github.com/rapidsai/cuvs/pull/652)) [@gforsyth](https://github.com/gforsyth)
- Revert docs builds to CI latest tag. ([#643](https://github.com/rapidsai/cuvs/pull/643)) [@bdice](https://github.com/bdice)
- Add float16 support in python for cagra/brute_force/ivf_pq and scalar quantizer ([#637](https://github.com/rapidsai/cuvs/pull/637)) [@benfred](https://github.com/benfred)
- Expose NN-Descent to C and Python ([#635](https://github.com/rapidsai/cuvs/pull/635)) [@benfred](https://github.com/benfred)
- Revert CUDA 12.8 shared workflow branch changes ([#630](https://github.com/rapidsai/cuvs/pull/630)) [@vyasr](https://github.com/vyasr)
- cuvs-java: Rework the api to be Java 21 friendly ([#628](https://github.com/rapidsai/cuvs/pull/628)) [@ChrisHegarty](https://github.com/ChrisHegarty)
- Build and test with CUDA 12.8.0 ([#621](https://github.com/rapidsai/cuvs/pull/621)) [@bdice](https://github.com/bdice)
- Add Scalar Quantization to the c and python apis ([#617](https://github.com/rapidsai/cuvs/pull/617)) [@benfred](https://github.com/benfred)
- Iteratively build graph index ([#612](https://github.com/rapidsai/cuvs/pull/612)) [@anaruse](https://github.com/anaruse)
- update pip devcontainers to UCX 1.18, small update-version.sh fixes ([#604](https://github.com/rapidsai/cuvs/pull/604)) [@jameslamb](https://github.com/jameslamb)
- Reduce CAGRA test runtime ([#602](https://github.com/rapidsai/cuvs/pull/602)) [@bdice](https://github.com/bdice)
- Revert "Temporarily skip CUDA 11 wheel CI" ([#601](https://github.com/rapidsai/cuvs/pull/601)) [@bdice](https://github.com/bdice)
- introduce libcuvs wheels ([#594](https://github.com/rapidsai/cuvs/pull/594)) [@jameslamb](https://github.com/jameslamb)
- Normalize whitespace ([#593](https://github.com/rapidsai/cuvs/pull/593)) [@bdice](https://github.com/bdice)
- Rename test to tests. ([#590](https://github.com/rapidsai/cuvs/pull/590)) [@bdice](https://github.com/bdice)
- Use cuda.bindings layout. ([#588](https://github.com/rapidsai/cuvs/pull/588)) [@bdice](https://github.com/bdice)
- run_cuvs_pytests.sh uses proper test dir ([#584](https://github.com/rapidsai/cuvs/pull/584)) [@robertmaynard](https://github.com/robertmaynard)
- expose col-major bfknn to python ([#575](https://github.com/rapidsai/cuvs/pull/575)) [@benfred](https://github.com/benfred)
- Run cuvs-bench pytests and end-to-end tests in CI ([#574](https://github.com/rapidsai/cuvs/pull/574)) [@dantegd](https://github.com/dantegd)
- Expose col-major pairwise distances to python ([#572](https://github.com/rapidsai/cuvs/pull/572)) [@benfred](https://github.com/benfred)
- Improve the performance of CAGRA new vector addition with the default params ([#569](https://github.com/rapidsai/cuvs/pull/569)) [@enp1s0](https://github.com/enp1s0)
- Improve filtering documentation ([#568](https://github.com/rapidsai/cuvs/pull/568)) [@lowener](https://github.com/lowener)
- Use GCC 13 in CUDA 12 conda builds. ([#567](https://github.com/rapidsai/cuvs/pull/567)) [@bdice](https://github.com/bdice)
- Allow brute_force::build to work on host matrix dataset ([#562](https://github.com/rapidsai/cuvs/pull/562)) [@benfred](https://github.com/benfred)
- FAISS with cuVS enabled in cuvs-bench ([#561](https://github.com/rapidsai/cuvs/pull/561)) [@tarang-jain](https://github.com/tarang-jain)
- Vamana build improvement and added docs ([#558](https://github.com/rapidsai/cuvs/pull/558)) [@bkarsin](https://github.com/bkarsin)
- Support raft's logger targets ([#557](https://github.com/rapidsai/cuvs/pull/557)) [@vyasr](https://github.com/vyasr)
- Get Breathe from conda again ([#554](https://github.com/rapidsai/cuvs/pull/554)) [@vyasr](https://github.com/vyasr)
- Check if nightlies have succeeded recently enough ([#548](https://github.com/rapidsai/cuvs/pull/548)) [@vyasr](https://github.com/vyasr)
- Add support for float16 to the python pairwise distance api ([#547](https://github.com/rapidsai/cuvs/pull/547)) [@benfred](https://github.com/benfred)
- Additional Distances for CAGRA C and Python API ([#546](https://github.com/rapidsai/cuvs/pull/546)) [@tarang-jain](https://github.com/tarang-jain)
- remove setup.cfg files, other packaging cleanup ([#544](https://github.com/rapidsai/cuvs/pull/544)) [@jameslamb](https://github.com/jameslamb)
- Fix CI for python cuvs_bench ([#541](https://github.com/rapidsai/cuvs/pull/541)) [@benfred](https://github.com/benfred)
- Update for raft logger changes ([#540](https://github.com/rapidsai/cuvs/pull/540)) [@vyasr](https://github.com/vyasr)
- Change brute_force api to match ivf*/cagra ([#536](https://github.com/rapidsai/cuvs/pull/536)) [@benfred](https://github.com/benfred)
- Branch 25.02 merge 24.12 ([#526](https://github.com/rapidsai/cuvs/pull/526)) [@benfred](https://github.com/benfred)
- Update cuda-python lower bounds to 12.6.2 / 11.8.5 ([#524](https://github.com/rapidsai/cuvs/pull/524)) [@bdice](https://github.com/bdice)
- Automatic adjustment of itopk size according to filtering rate ([#509](https://github.com/rapidsai/cuvs/pull/509)) [@anaruse](https://github.com/anaruse)
- prefer system install of UCX in devcontainers ([#501](https://github.com/rapidsai/cuvs/pull/501)) [@jameslamb](https://github.com/jameslamb)
- Adapt to rmm logger changes ([#499](https://github.com/rapidsai/cuvs/pull/499)) [@vyasr](https://github.com/vyasr)
- Require approval to run CI on draft PRs ([#498](https://github.com/rapidsai/cuvs/pull/498)) [@bdice](https://github.com/bdice)
- Remove RAFT BUILD_ANN_BENCH option ([#497](https://github.com/rapidsai/cuvs/pull/497)) [@bdice](https://github.com/bdice)
- Update example code fetching rapids-cmake to use CUVS instead of RAFT ([#493](https://github.com/rapidsai/cuvs/pull/493)) [@bdice](https://github.com/bdice)
- Improve multi-CTA algorithm ([#492](https://github.com/rapidsai/cuvs/pull/492)) [@anaruse](https://github.com/anaruse)
- Add filtering for CAGRA to C API ([#452](https://github.com/rapidsai/cuvs/pull/452)) [@ajit283](https://github.com/ajit283)
- Initial cut for a cuVS Java API ([#450](https://github.com/rapidsai/cuvs/pull/450)) [@chatman](https://github.com/chatman)
- Add breaking change workflow trigger ([#442](https://github.com/rapidsai/cuvs/pull/442)) [@AyodeAwe](https://github.com/AyodeAwe)
- Expose `extend()` in C API ([#276](https://github.com/rapidsai/cuvs/pull/276)) [@ajit283](https://github.com/ajit283)
- Go API - [WIP] ([#212](https://github.com/rapidsai/cuvs/pull/212)) [@ajit283](https://github.com/ajit283)

# cuvs 24.12.00 (11 Dec 2024)

## 🚨 Breaking Changes
Expand Down
10 changes: 9 additions & 1 deletion build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -18,14 +18,15 @@ ARGS=$*
# scripts, and that this script resides in the repo dir!
REPODIR=$(cd $(dirname $0); pwd)

VALIDARGS="clean libcuvs python rust java docs tests bench-ann examples --uninstall -v -g -n --compile-static-lib --allgpuarch --no-mg --no-cpu --cpu-only --no-shared-libs --no-nvtx --show_depr_warn --incl-cache-stats --time -h"
VALIDARGS="clean libcuvs python rust go java docs tests bench-ann examples --uninstall -v -g -n --compile-static-lib --allgpuarch --no-mg --no-cpu --cpu-only --no-shared-libs --no-nvtx --show_depr_warn --incl-cache-stats --time -h"
HELP="$0 [<target> ...] [<flag> ...] [--cmake-args=\"<args>\"] [--cache-tool=<tool>] [--limit-tests=<targets>] [--limit-bench-ann=<targets>] [--build-metrics=<filename>]
where <target> is:
clean - remove all existing build artifacts and configuration (start over)
libcuvs - build the cuvs C++ code only. Also builds the C-wrapper library
around the C++ code.
python - build the cuvs Python package
rust - build the cuvs Rust bindings
go - build the cuvs Go bindings
java - build the cuvs Java bindings
docs - build the documentation
tests - build the tests
Expand Down Expand Up @@ -447,6 +448,13 @@ if (( ${NUMARGS} == 0 )) || hasArg rust; then
cargo test
fi

# Build the cuvs Go bindings
if (( ${NUMARGS} == 0 )) || hasArg go; then
cd ${REPODIR}/go
go build ./...
go test ./...
fi

# Build the cuvs Java bindings
if (( ${NUMARGS} == 0 )) || hasArg java; then
if ! hasArg libcuvs; then
Expand Down
20 changes: 8 additions & 12 deletions ci/build_docs.sh
Original file line number Diff line number Diff line change
Expand Up @@ -3,17 +3,23 @@

set -euo pipefail

rapids-logger "Downloading artifacts from previous jobs"
CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python)

rapids-logger "Create test conda environment"
. /opt/conda/etc/profile.d/conda.sh

RAPIDS_VERSION="$(rapids-version)"
RAPIDS_VERSION_MAJOR_MINOR="$(rapids-version-major-minor)"
export RAPIDS_VERSION_MAJOR_MINOR

rapids-dependency-file-generator \
--output conda \
--file-key docs \
--matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
--matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" \
--prepend-channel "${CPP_CHANNEL}" \
--prepend-channel "${PYTHON_CHANNEL}" \
| tee env.yaml

rapids-mamba-retry env create --yes -f env.yaml -n docs

Expand All @@ -25,16 +31,6 @@ set -eu

rapids-print-env

rapids-logger "Downloading artifacts from previous jobs"
CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python)

rapids-mamba-retry install \
--channel "${CPP_CHANNEL}" \
--channel "${PYTHON_CHANNEL}" \
"libcuvs=${RAPIDS_VERSION}" \
"cuvs=${RAPIDS_VERSION}"

RAPIDS_DOCS_DIR="$(mktemp -d)"
export RAPIDS_DOCS_DIR

Expand Down
34 changes: 34 additions & 0 deletions ci/build_go.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
#!/bin/bash
# Copyright (c) 2024, NVIDIA CORPORATION.

set -euo pipefail

rapids-logger "Downloading artifacts from previous jobs"
CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)

rapids-logger "Create test conda environment"
. /opt/conda/etc/profile.d/conda.sh

rapids-dependency-file-generator \
--output conda \
--file-key go \
--matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" \
--prepend-channel "${CPP_CHANNEL}" \
| tee env.yaml

rapids-mamba-retry env create --yes -f env.yaml -n go

# seeing failures on activating the environment here on unbound locals
# apply workaround from https://github.com/conda/conda/issues/8186#issuecomment-532874667
set +eu
conda activate go
set -eu

rapids-print-env

export CGO_CFLAGS="-I${CONDA_PREFIX}/include"
export CGO_LDFLAGS="-L${CONDA_PREFIX}/lib -lcudart -lcuvs -lcuvs_c"
export LD_LIBRARY_PATH="$CONDA_PREFIX/lib:$LD_LIBRARY_PATH"
export CC=clang

bash ./build.sh go
2 changes: 2 additions & 0 deletions ci/release/update-version.sh
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,9 @@ echo "${NEXT_FULL_TAG}" > VERSION
DEPENDENCIES=(
dask-cuda
cuvs
cuvs-bench
libcuvs
libcuvs-tests
libraft
librmm
pylibraft
Expand Down
Loading

0 comments on commit 07948cb

Please sign in to comment.