From 20595c4e9c204c6fb8a5d432157a7ddc0dbe1441 Mon Sep 17 00:00:00 2001 From: Ayush Dattagupta Date: Wed, 20 Mar 2024 17:25:47 -0700 Subject: [PATCH 1/8] Add workflow for cpu pytests Signed-off-by: Ayush Dattagupta --- .github/workflows/test.yml | 37 +++++++++++++++++++++++++++++++++++++ conftest.py | 15 +++++++++++++++ 2 files changed, 52 insertions(+) create mode 100644 .github/workflows/test.yml create mode 100644 conftest.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..40f629dd --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,37 @@ +name: Test Python package +on: + push: + branches: + - main + pull_request: + workflow_dispatch: + +# When this workflow is queued, automatically cancel any previous running +# or pending jobs from the same branch +concurrency: + group: test-${{ github.ref }} + cancel-in-progress: true + +jobs: + build_and_test: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest] + python-version: ["3.8", "3.9", "3.10"] + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install NeMo-Curator and pytest + # Remove installing pytest when optional test dependencies are added to setup.py + run: | + pip install --extra-index-url https://pypi.nvidia.com . + pip install pytest + - name: Run tests + run: | + python -m pytest -v --cpu + + diff --git a/conftest.py b/conftest.py new file mode 100644 index 00000000..451ae5af --- /dev/null +++ b/conftest.py @@ -0,0 +1,15 @@ +import pytest + + +def pytest_addoption(parser): + parser.addoption( + "--cpu", action="store_true", default=False, help="Run tests without gpu marker" + ) + + +def pytest_collection_modifyitems(config, items): + if config.getoption("--cpu"): + skip_gpu = pytest.mark.skip(reason="Skipping GPU tests") + for item in items: + if "gpu" in item.keywords: + item.add_marker(skip_gpu) From b035dd764f4df78b68f8f2249b515f97a49c4e40 Mon Sep 17 00:00:00 2001 From: Ayush Dattagupta Date: Wed, 20 Mar 2024 17:34:27 -0700 Subject: [PATCH 2/8] Install wheel to fix fasttext import Signed-off-by: Ayush Dattagupta --- .github/workflows/test.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 40f629dd..976b5f36 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -27,7 +27,10 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install NeMo-Curator and pytest # Remove installing pytest when optional test dependencies are added to setup.py + # Installing wheel beforehand due to fasttext issue: + # https://github.com/facebookresearch/fastText/issues/512#issuecomment-1837367666 run: | + pip install wheel pip install --extra-index-url https://pypi.nvidia.com . pip install pytest - name: Run tests From b7aa79052447f41cc9a38c45669b589a0812f653 Mon Sep 17 00:00:00 2001 From: Ayush Dattagupta Date: Wed, 20 Mar 2024 17:38:50 -0700 Subject: [PATCH 3/8] Omit python 3.8, do not fast fail Signed-off-by: Ayush Dattagupta --- .github/workflows/test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 976b5f36..b6ac8e68 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -16,9 +16,10 @@ jobs: build_and_test: runs-on: ${{ matrix.os }} strategy: + fail-fast: false matrix: os: [ubuntu-latest] - python-version: ["3.8", "3.9", "3.10"] + python-version: ["3.9", "3.10"] steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} From 4738bdb5f1066a1acbe52b5ade3dc01ecb57a964 Mon Sep 17 00:00:00 2001 From: Ayush Dattagupta Date: Wed, 20 Mar 2024 17:46:25 -0700 Subject: [PATCH 4/8] Check if updating setuptools/pip changes cython errors Signed-off-by: Ayush Dattagupta --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b6ac8e68..04321cd3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -31,7 +31,7 @@ jobs: # Installing wheel beforehand due to fasttext issue: # https://github.com/facebookresearch/fastText/issues/512#issuecomment-1837367666 run: | - pip install wheel + pip install wheel setuptools pip --upgrade pip install --extra-index-url https://pypi.nvidia.com . pip install pytest - name: Run tests From 3e6aef83a5b4d5716f1c5e330408dcd3241da7a0 Mon Sep 17 00:00:00 2001 From: Ayush Dattagupta Date: Wed, 20 Mar 2024 17:50:33 -0700 Subject: [PATCH 5/8] Explicitly install cython Signed-off-by: Ayush Dattagupta --- .github/workflows/test.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 04321cd3..d4d60ac2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -27,11 +27,13 @@ jobs: with: python-version: ${{ matrix.python-version }} - name: Install NeMo-Curator and pytest - # Remove installing pytest when optional test dependencies are added to setup.py + # TODO: Remove pytest when optional test dependencies are added to setup.py + # Installing wheel beforehand due to fasttext issue: # https://github.com/facebookresearch/fastText/issues/512#issuecomment-1837367666 + # Explicitly install cython: https://github.com/VKCOM/YouTokenToMe/issues/94 run: | - pip install wheel setuptools pip --upgrade + pip install wheel cython pip install --extra-index-url https://pypi.nvidia.com . pip install pytest - name: Run tests From 41a9f78490fe3556afbcb8fd1684fd5b614254e2 Mon Sep 17 00:00:00 2001 From: Ayush Dattagupta Date: Wed, 20 Mar 2024 18:05:21 -0700 Subject: [PATCH 6/8] Try freeing up space before install Signed-off-by: Ayush Dattagupta --- .github/workflows/test.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d4d60ac2..b7002772 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -22,6 +22,13 @@ jobs: python-version: ["3.9", "3.10"] steps: - uses: actions/checkout@v4 + # TODO: Should not be needed after GPU imports are optional + - name: Optionally free up space on Ubuntu + run: | + sudo rm -rf /usr/share/dotnet + sudo rm -rf /opt/ghc + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: From 46bd8d29c3a3b20972b2aa9623a202d6b3be2fb1 Mon Sep 17 00:00:00 2001 From: Ayush Dattagupta Date: Wed, 20 Mar 2024 18:19:15 -0700 Subject: [PATCH 7/8] Try rapids_no_initialize Signed-off-by: Ayush Dattagupta --- .github/workflows/test.yml | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b7002772..c0120af3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -22,13 +22,6 @@ jobs: python-version: ["3.9", "3.10"] steps: - uses: actions/checkout@v4 - # TODO: Should not be needed after GPU imports are optional - - name: Optionally free up space on Ubuntu - run: | - sudo rm -rf /usr/share/dotnet - sudo rm -rf /opt/ghc - sudo rm -rf "/usr/local/share/boost" - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: @@ -41,10 +34,11 @@ jobs: # Explicitly install cython: https://github.com/VKCOM/YouTokenToMe/issues/94 run: | pip install wheel cython - pip install --extra-index-url https://pypi.nvidia.com . + pip install --no-cache-dir --extra-index-url https://pypi.nvidia.com . pip install pytest - name: Run tests + # TODO: Remove env variable when gpu dependencies are optional run: | - python -m pytest -v --cpu + RAPIDS_NO_INITIALIZE=true python -m pytest -v --cpu From 2cdc937c4ae239d9fced275cdb3dd4c91c5054f9 Mon Sep 17 00:00:00 2001 From: Ayush Dattagupta Date: Thu, 21 Mar 2024 14:56:48 -0700 Subject: [PATCH 8/8] remove python 3.9 testing for now Signed-off-by: Ayush Dattagupta --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c0120af3..375df5b9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,7 +19,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest] - python-version: ["3.9", "3.10"] + python-version: ["3.10"] steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} @@ -39,6 +39,6 @@ jobs: - name: Run tests # TODO: Remove env variable when gpu dependencies are optional run: | - RAPIDS_NO_INITIALIZE=true python -m pytest -v --cpu + RAPIDS_NO_INITIALIZE=1 python -m pytest -v --cpu