add full CI for wholegraph (#58)

This PR introduces full end-to-end CI for wholegraph.

## Notes for Reviewers

Most of these changes were pulled from a mix of the following:

* #53
* rapidsai/wholegraph#230
* https://github.com/rapidsai/wholegraph

Authors:
  - James Lamb (https://github.com/jameslamb)

Approvers:
  - Alex Barghi (https://github.com/alexbarghi-nv)
  - Jake Awe (https://github.com/AyodeAwe)

URL: #58

.github/workflows/pr.yaml

@@ -14,6 +14,12 @@ jobs:
     needs:
       - changed-files
       - checks
+      - conda-cpp-build
+      - conda-cpp-tests
+      - conda-python-build
+      - conda-python-tests
+      - wheel-build-pylibwholegraph
+      - wheel-tests-pylibwholegraph
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
     if: always()
@@ -54,3 +60,47 @@ jobs:
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
     with:
       enable_check_generated_files: false
+  conda-cpp-build:
+    needs: checks
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    with:
+      build_type: pull-request
+      node_type: cpu32
+  conda-cpp-tests:
+    needs: [conda-cpp-build, changed-files]
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_cpp
+    with:
+      build_type: pull-request
+  conda-python-build:
+    needs: conda-cpp-build
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    with:
+      build_type: pull-request
+  conda-python-tests:
+    needs: [conda-python-build, changed-files]
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python
+    with:
+      build_type: pull-request
+  wheel-build-pylibwholegraph:
+    needs: checks
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    with:
+      build_type: pull-request
+      script: ci/build_wheel_pylibwholegraph.sh
+      node_type: cpu32
+  wheel-tests-pylibwholegraph:
+    needs: [wheel-build-pylibwholegraph, changed-files]
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python
+    with:
+      build_type: pull-request
+      script: ci/test_wheel_pylibwholegraph.sh
+      matrix_filter: map(select(.ARCH == "amd64"))

build.sh

@@ -26,6 +26,9 @@ VALIDARGS="
    uninstall
    cugraph-pyg
    cugraph-dgl
+   pylibwholegraph
+   libwholegraph
+   tests
    docs
    all
    -v
@@ -44,6 +47,9 @@ HELP="$0 [<target> ...] [<flag> ...]
    uninstall                  - uninstall libcugraph and cugraph from a prior build/install (see also -n)
    cugraph-pyg                - build the cugraph-pyg Python package
    cugraph-dgl                - build the cugraph-dgl extensions for DGL
+   pylibwholegraph            - build the pylibwholegraph Python package
+   libwholegraph              - build the libwholegraph library
+   tests                      - build the C++ tests
    docs                       - build the docs
    all                        - build everything
  and <flag> is:
@@ -55,23 +61,27 @@ HELP="$0 [<target> ...] [<flag> ...]
    --clean                    - clean an individual target (note: to do a complete rebuild, use the clean target described above)
    -h                         - print this text
 
- default action (no args) is to build and install 'cugraph-pyg' then 'cugraph-dgl' then 'wholegraph' targets
+ default action (no args) is to build and install 'libwholegraph' then 'pylibwholegraph' then 'cugraph-pyg' then 'cugraph-dgl'
 
 "
 
 CUGRAPH_PYG_BUILD_DIR=${REPODIR}/python/cugraph-pyg/build
 CUGRAPH_DGL_BUILD_DIR=${REPODIR}/python/cugraph-dgl/build
+PYLIBWHOLEGRAPH_BUILD_DIR=${REPODIR}/python/pylibwholegraph/build
+LIBWHOLEGRAPH_BUILD_DIR=${REPODIR}/cpp/build
 
 BUILD_DIRS="${CUGRAPH_PYG_BUILD_DIR}
             ${CUGRAPH_DGL_BUILD_DIR}
+            ${PYLIBWHOLEGRAPH_BUILD_DIR}
+            ${LIBWHOLEGRAPH_BUILD_DIR}
 "
 
 # Set defaults for vars modified by flags to this script
 VERBOSE_FLAG=""
 BUILD_TYPE=Release
 INSTALL_TARGET="--target install"
 BUILD_ALL_GPU_ARCH=0
-PYTHON_ARGS_FOR_INSTALL="-m pip install --no-build-isolation --no-deps"
+PYTHON_ARGS_FOR_INSTALL="-m pip install --no-build-isolation --no-deps --config-settings rapidsai.disable-cuda=true"
 
 # Set defaults for vars that may not have been defined externally
 #  FIXME: if PREFIX is not set, check CONDA_PREFIX, but there is no fallback
@@ -132,13 +142,30 @@ if hasArg --pydevelop; then
     PYTHON_ARGS_FOR_INSTALL="${PYTHON_ARGS_FOR_INSTALL} -e"
 fi
 
+if hasArg tests; then
+    BUILD_TESTS=ON
+else
+    BUILD_TESTS=OFF
+fi
+
 # If clean or uninstall targets given, run them prior to any other steps
 if hasArg uninstall; then
-    # uninstall cugraph and pylibcugraph installed from a prior install
+    if [[ "$INSTALL_PREFIX" != "" ]]; then
+        rm -rf ${INSTALL_PREFIX}/include/wholememory
+        rm -f ${INSTALL_PREFIX}/lib/libwholegraph.so
+        rm -rf ${INSTALL_PREFIX}/lib/cmake/wholegraph
+    fi
+    # This may be redundant given the above, but can also be used in case
+    # there are other installed files outside of the locations above.
+    if [ -e ${LIBWHOLEGRAPH_BUILD_DIR}/install_manifest.txt ]; then
+        xargs rm -f < ${LIBWHOLEGRAPH_BUILD_DIR}/install_manifest.txt > /dev/null 2>&1
+    fi
+
+    # uninstall cugraph-dgl/cugraph-pyg/wholegraph installed from a prior install
     # FIXME: if multiple versions of these packages are installed, this only
     # removes the latest one and leaves the others installed. build.sh uninstall
     # can be run multiple times to remove all of them, but that is not obvious.
-    pip uninstall -y  cugraph-dgl cugraph-pyg
+    pip uninstall -y  cugraph-dgl cugraph-pyg pylibwholegraph libwholegraph
 fi
 
 if hasArg clean; then
@@ -164,6 +191,58 @@ if hasArg clean; then
 fi
 
 ################################################################################
+# Build and install the libwholegraph library
+if hasArg libwholegraph || buildDefault || hasArg all ; then
+
+    # set values based on flags
+    if (( ${BUILD_ALL_GPU_ARCH} == 0 )); then
+        WHOLEGRAPH_CMAKE_CUDA_ARCHITECTURES="${WHOLEGRAPH_CMAKE_CUDA_ARCHITECTURES:=NATIVE}"
+        echo "Building for the architecture of the GPU in the system..."
+    else
+        WHOLEGRAPH_CMAKE_CUDA_ARCHITECTURES="70-real;75-real;80-real;86-real;90"
+        echo "Building for *ALL* supported GPU architectures..."
+    fi
+
+    cmake -S ${REPODIR}/cpp -B ${LIBWHOLEGRAPH_BUILD_DIR} \
+          -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} \
+          -DCMAKE_CUDA_ARCHITECTURES=${WHOLEGRAPH_CMAKE_CUDA_ARCHITECTURES} \
+          -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
+          -DCMAKE_PREFIX_PATH=${INSTALL_PREFIX} \
+          -DCMAKE_MESSAGE_LOG_LEVEL=VERBOSE \
+          -DBUILD_TESTS=${BUILD_TESTS} \
+          -DBUILD_WITH_NVSHMEM=${BUILD_WITH_NVSHMEM}
+
+    cd ${LIBWHOLEGRAPH_BUILD_DIR}
+
+    if ! hasArg --compile-cmd; then
+        ## Build and (optionally) install library + tests
+        cmake --build . -j${PARALLEL_LEVEL} ${INSTALL_TARGET} ${VERBOSE_FLAG}
+    fi
+fi
+
+# Build and install the pylibwholegraph Python package
+if hasArg pylibwholegraph || buildDefault || hasArg all; then
+    if hasArg --clean; then
+        cleanPythonDir ${REPODIR}/python/pylibwholegraph
+    fi
+
+    # setup.py and cmake reference an env var LIBWHOLEGRAPH_DIR to find the
+    # libwholegraph package (cmake).
+    # If not set by the user, set it to LIBWHOLEGRAPH_BUILD_DIR
+    LIBWHOLEGRAPH_DIR=${LIBWHOLEGRAPH_DIR:=${LIBWHOLEGRAPH_BUILD_DIR}}
+    if ! hasArg --compile-cmd; then
+        cd ${REPODIR}/python/pylibwholegraph
+        env LIBWHOLEGRAPH_DIR=${LIBWHOLEGRAPH_DIR} \
+        SKBUILD_CMAKE_ARGS="-DCMAKE_BUILD_TYPE=${BUILD_TYPE}" python ${PYTHON_ARGS_FOR_INSTALL} \
+            .
+    else
+        # just invoke cmake without going through scikit-build-core
+        env LIBWHOLEGRAPH_DIR=${LIBWHOLEGRAPH_DIR} \
+        cmake -S ${REPODIR}/python/pylibwholegraph -B ${REPODIR}/python/pylibwholegraph/build \
+           -DCMAKE_BUILD_TYPE=${BUILD_TYPE}
+    fi
+fi
+
 # Build and install the cugraph-pyg Python package
 if hasArg cugraph-pyg || buildDefault || hasArg all; then
     if hasArg --clean; then

ci/build_cpp.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+# Copyright (c) 2022-2024, NVIDIA CORPORATION.
+
+set -euo pipefail
+
+rapids-configure-conda-channels
+
+source rapids-configure-sccache
+
+source rapids-date-string
+
+export CMAKE_GENERATOR=Ninja
+
+rapids-print-env
+
+version=$(rapids-generate-version)
+
+rapids-logger "Begin cpp build"
+
+RAPIDS_PACKAGE_VERSION=${version} rapids-conda-retry mambabuild conda/recipes/libwholegraph
+
+rapids-upload-conda-to-s3 cpp

ci/build_python.sh

@@ -15,41 +15,14 @@ rapids-print-env
 
 CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
 
-version=$(rapids-generate-version)
-git_commit=$(git rev-parse HEAD)
-export RAPIDS_PACKAGE_VERSION=${version}
-echo "${version}" > VERSION
-
-rapids-logger "Begin py build"
-
-package_dir="python"
-for package_name in cugraph-pyg cugraph-dgl; do
-  underscore_package_name=$(echo "${package_name}" | tr "-" "_")
-  sed -i "/^__git_commit__/ s/= .*/= \"${git_commit}\"/g" "${package_dir}/${package_name}/${underscore_package_name}/_version.py"
-done
-
-RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}"
-
-if [[ ${RAPIDS_CUDA_MAJOR} == "11" ]]; then
-  # Only CUDA 11 is supported right now due to PyTorch requirement.
-  rapids-conda-retry mambabuild \
-    --no-test \
-    --channel "${CPP_CHANNEL}" \
-    --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \
-    --channel pyg \
-    --channel pytorch \
-    --channel pytorch-nightly \
-    conda/recipes/cugraph-pyg
-
-  # Only CUDA 11 is supported right now due to PyTorch requirement.
-  rapids-conda-retry mambabuild \
-    --no-test \
-    --channel "${CPP_CHANNEL}" \
-    --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \
-    --channel dglteam \
-    --channel pytorch \
-    --channel pytorch-nightly \
-    conda/recipes/cugraph-dgl
-fi
+rapids-generate-version > ./VERSION
+
+# TODO: Remove `--no-test` flags once importing on a CPU
+# node works correctly
+rapids-logger "Begin pylibwholegraph build"
+RAPIDS_PACKAGE_VERSION=$(head -1 ./VERSION) rapids-conda-retry mambabuild \
+  --no-test \
+  --channel "${CPP_CHANNEL}" \
+  conda/recipes/pylibwholegraph
 
 rapids-upload-conda-to-s3 python

ci/build_wheel.sh

@@ -7,62 +7,52 @@ package_name=$1
 package_dir=$2
 underscore_package_name=$(echo "${package_name}" | tr "-" "_")
 
+# The set of shared libraries that should be packaged differs by project.
+#
+# Capturing that here in argument-parsing to allow this build_wheel.sh
+# script to be re-used by all wheel builds in the project.
+case "${package_dir}" in
+  python/pylibwholegraph)
+    EXCLUDE_ARGS=(
+        --exclude libcuda.so.1
+        --exclude libnvidia-ml.so.1
+    )
+  ;;
+  *)
+    EXCLUDE_ARGS=()
+  ;;
+esac
+
 source rapids-configure-sccache
 source rapids-date-string
 
-version=$(rapids-generate-version)
-git_commit=$(git rev-parse HEAD)
+rapids-generate-version > ./VERSION
 
 RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
 
-# This is the version of the suffix with a preceding hyphen. It's used
-# everywhere except in the final wheel name.
-PACKAGE_CUDA_SUFFIX="-${RAPIDS_PY_CUDA_SUFFIX}"
-
-# Patch project metadata files to include the CUDA version suffix and version override.
-version_package_name="$underscore_package_name"
-
-pyproject_file="${package_dir}/pyproject.toml"
-version_file="${package_dir}/${version_package_name}/_version.py"
-
-sed -i "s/name = \"${package_name}\"/name = \"${package_name}${PACKAGE_CUDA_SUFFIX}\"/g" ${pyproject_file}
-echo "${version}" > VERSION
-sed -i "/^__git_commit__ / s/= .*/= \"${git_commit}\"/g" ${version_file}
-
-# For nightlies we want to ensure that we're pulling in alphas as well. The
-# easiest way to do so is to augment the spec with a constraint containing a
-# min alpha version that doesn't affect the version bounds but does allow usage
-# of alpha versions for that dependency without --pre
-alpha_spec=''
-if ! rapids-is-release-build; then
-    alpha_spec=',>=0.0.0a0'
-fi
-
-for dep in rmm cudf cugraph raft-dask pylibcugraph pylibcugraphops pylibwholegraph pylibraft ucx-py; do
-    sed -r -i "s/${dep}==(.*)\"/${dep}${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file}
-done
-
-# dask-cuda & rapids-dask-dependency doesn't get a suffix, but it does get an alpha spec.
-for dep in dask-cuda rapids-dask-dependency; do
-    sed -r -i "s/${dep}==(.*)\"/${dep}==\1${alpha_spec}\"/g" ${pyproject_file}
-done
-
-
-if [[ $PACKAGE_CUDA_SUFFIX == "-cu12" ]]; then
-    sed -i "s/cupy-cuda11x/cupy-cuda12x/g" ${pyproject_file}
-fi
-
 cd "${package_dir}"
 
-python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check
+rapids-logger "Building '${package_name}' wheel"
+python -m pip wheel \
+    -w dist \
+    -v \
+    --no-deps \
+    --disable-pip-version-check \
+    .
+
+sccache --show-adv-stats
 
 # pure-python packages should be marked as pure, and not have auditwheel run on them.
 if [[ ${package_name} == "cugraph-dgl" ]] || \
    [[ ${package_name} == "cugraph-pyg" ]]; then
-    RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-upload-wheels-to-s3 dist
+    RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-upload-wheels-to-s3 python dist
 else
-    # presumably WholeGraph when we add it
+
     mkdir -p final_dist
-    python -m auditwheel repair -w final_dist dist/*
+    python -m auditwheel repair \
+        "${EXCLUDE_ARGS[@]}" \
+        -w final_dist \
+        dist/*
+
     RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 final_dist
 fi

ci/build_wheel_pylibwholegraph.sh

@@ -0,0 +1,9 @@
+
+#!/bin/bash
+# Copyright (c) 2024, NVIDIA CORPORATION.
+
+set -euo pipefail
+
+export SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF;-DBUILD_SHARED_LIBS=OFF;-DCMAKE_MESSAGE_LOG_LEVEL=VERBOSE;-DCUDA_STATIC_RUNTIME=ON;-DWHOLEGRAPH_BUILD_WHEELS=ON"
+
+./ci/build_wheel.sh pylibwholegraph python/pylibwholegraph

ci/run_ctests.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+# Copyright (c) 2024, NVIDIA CORPORATION.
+
+set -euo pipefail
+
+# Support customizing the ctests' install location
+cd "${INSTALL_PREFIX:-${CONDA_PREFIX:-/usr}}/bin/gtests/libwholegraph/"
+
+find . -type f -executable -print0 | xargs -0 -r -t -n1 -P1 sh -c 'exec "$0"';