From 2cd02f35e2bfd9ac2b6d846f43dcad07d3fd6561 Mon Sep 17 00:00:00 2001 From: Ayush Dattagupta Date: Thu, 21 Mar 2024 15:21:07 -0700 Subject: [PATCH] Add workflow for running cpu pytests (#13) * Add workflow for cpu pytests Signed-off-by: Ayush Dattagupta * Install wheel to fix fasttext import Signed-off-by: Ayush Dattagupta * Omit python 3.8, do not fast fail Signed-off-by: Ayush Dattagupta * Check if updating setuptools/pip changes cython errors Signed-off-by: Ayush Dattagupta * Explicitly install cython Signed-off-by: Ayush Dattagupta * Try freeing up space before install Signed-off-by: Ayush Dattagupta * Try rapids_no_initialize Signed-off-by: Ayush Dattagupta * remove python 3.9 testing for now Signed-off-by: Ayush Dattagupta --------- Signed-off-by: Ayush Dattagupta --- .github/workflows/test.yml | 44 ++++++++++++++++++++++++++++++++++++++ conftest.py | 15 +++++++++++++ 2 files changed, 59 insertions(+) create mode 100644 .github/workflows/test.yml create mode 100644 conftest.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..375df5b9 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,44 @@ +name: Test Python package +on: + push: + branches: + - main + pull_request: + workflow_dispatch: + +# When this workflow is queued, automatically cancel any previous running +# or pending jobs from the same branch +concurrency: + group: test-${{ github.ref }} + cancel-in-progress: true + +jobs: + build_and_test: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + python-version: ["3.10"] + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install NeMo-Curator and pytest + # TODO: Remove pytest when optional test dependencies are added to setup.py + + # Installing wheel beforehand due to fasttext issue: + # https://github.com/facebookresearch/fastText/issues/512#issuecomment-1837367666 + # Explicitly install cython: https://github.com/VKCOM/YouTokenToMe/issues/94 + run: | + pip install wheel cython + pip install --no-cache-dir --extra-index-url https://pypi.nvidia.com . + pip install pytest + - name: Run tests + # TODO: Remove env variable when gpu dependencies are optional + run: | + RAPIDS_NO_INITIALIZE=1 python -m pytest -v --cpu + + diff --git a/conftest.py b/conftest.py new file mode 100644 index 00000000..451ae5af --- /dev/null +++ b/conftest.py @@ -0,0 +1,15 @@ +import pytest + + +def pytest_addoption(parser): + parser.addoption( + "--cpu", action="store_true", default=False, help="Run tests without gpu marker" + ) + + +def pytest_collection_modifyitems(config, items): + if config.getoption("--cpu"): + skip_gpu = pytest.mark.skip(reason="Skipping GPU tests") + for item in items: + if "gpu" in item.keywords: + item.add_marker(skip_gpu)