From 358f697f584c21b2ee09b82e9ca5746e1f74d48f Mon Sep 17 00:00:00 2001 From: Pierre-antoine Comby Date: Mon, 3 Mar 2025 13:13:57 +0100 Subject: [PATCH 1/3] fix: use venv in all CI. --- .github/workflows/test-ci.yml | 60 +++++++++++++++++++++++------------ 1 file changed, 40 insertions(+), 20 deletions(-) diff --git a/.github/workflows/test-ci.yml b/.github/workflows/test-ci.yml index 4720af6ae..d105f9884 100644 --- a/.github/workflows/test-ci.yml +++ b/.github/workflows/test-ci.yml @@ -16,6 +16,9 @@ env: PYTHON_VERSION: "3.10" BART_VERSION: "0.8.00" ref_backend: "finufft" + create_venv: "python -m venv $RUNNER_WORKSPACE/venv/" + activate_venv_venv: "source $RUNNER_WORKSPACE/venv/bin/activate" + setup_cuda: "export CUDA_BIN_PATH=/usr/local/cuda-12.1/;export PATH=/usr/local/cuda-12.1/bin/:${PATH};export LD_LIBRARY_PATH=/usr/local/cuda-12.1/lib64/:${LD_LIBRARY_PATH}" jobs: test-cpu: @@ -44,6 +47,9 @@ jobs: - name: Install Python Deps shell: bash run: | + python --version + ${{env.create_venv}} + ${{env.activate_venv_venv}} python -m pip install --upgrade pip python -m pip install -e .[test] @@ -51,21 +57,28 @@ jobs: if: ${{ matrix.backend == 'pynfft' || env.ref_backend == 'pynfft' }} shell: bash run: | + ${{env.activate_venv}} python -m pip install "pynfft2>=1.4.3" - name: Install pynufft if: ${{ matrix.backend == 'pynufft-cpu' || env.ref_backend == 'pynufft-cpu' }} - run: python -m pip install pynufft + run: | + ${{env.activate_venv}} + python -m pip install pynufft - name: Install finufft if: ${{ matrix.backend == 'finufft' || env.ref_backend == 'finufft'}} shell: bash - run: python -m pip install finufft + run: | + ${{env.activate_venv}} + python -m pip install finufft - name: Install Sigpy if: ${{ matrix.backend == 'sigpy' || env.ref_backend == 'sigpy'}} shell: bash - run: python -m pip install sigpy + run: | + ${{env.activate_venv}} + python -m pip install sigpy - name: Install BART if: ${{ matrix.backend == 'bart' || env.ref_backend == 'bart'}} @@ -81,12 +94,15 @@ jobs: - name: Install torchkbnufft-cpu if: ${{ matrix.backend == 'torchkbnufft-cpu' || env.ref_backend == 'torchkbnufft-cpu'}} - run: python -m pip install torchkbnufft + run: | + ${{env.activate_venv}} + python -m pip install torchkbnufft - name: Run Tests shell: bash run: | + ${{env.activate_venv}} export COVERAGE_FILE=coverage_${{ matrix.backend }} pytest -k='operators' --backend ${{ matrix.backend }} --ref ${{ env.ref_backend }} --cov --disable-pytest-warnings --cov-branch --cov-report=term - name: Upload coverage @@ -111,10 +127,8 @@ jobs: - name: Install mri-nufft and finufft shell: bash run: | - cd $RUNNER_WORKSPACE - python --version - python -m venv venv - source $RUNNER_WORKSPACE/venv/bin/activate + ${{ env.create_venv }} + ${{ env.activate_venv }} pip install --upgrade pip wheel pip install -e mri-nufft[test] pip install cupy-cuda12x finufft "numpy<2.0" @@ -123,16 +137,14 @@ jobs: shell: bash if: ${{ matrix.backend != 'tensorflow'}} run: | - source $RUNNER_WORKSPACE/venv/bin/activate + source $RUNNER_WORKSPACE/venv/binactivate_venv pip install torch - name: Install backend shell: bash run: | - source $RUNNER_WORKSPACE/venv/bin/activate - export CUDA_BIN_PATH=/usr/local/cuda-12.1/ - export PATH=/usr/local/cuda-12.1/bin/:${PATH} - export LD_LIBRARY_PATH=/usr/local/cuda-12.1/lib64/:${LD_LIBRARY_PATH} + ${{ env.activate_venv }} + ${{ env.setup_cuda }} if [[ ${{ matrix.backend }} == "torchkbnufft-gpu" ]]; then pip install torchkbnufft elif [[ ${{ matrix.backend }} == "tensorflow" ]]; then @@ -146,8 +158,8 @@ jobs: - name: Run Tests shell: bash run: | + ${{ env.activate_venv }} cd $RUNNER_WORKSPACE/mri-nufft - source $RUNNER_WORKSPACE/venv/bin/activate export COVERAGE_FILE=coverage_${{ matrix.backend }} python -m pytest -k='operators' --ref ${{ env.ref_backend }} --backend ${{ matrix.backend }} --disable-pytest-warnings --cov --cov-branch --cov-report=term @@ -205,21 +217,23 @@ jobs: - name: Install Python deps shell: bash run: | + ${{ env.create_venv }} + ${{ env.activate_venv }} python -m pip install --upgrade pip python -m pip install -e .[extra,test,dev] python -m pip install finufft pooch brainweb-dl torch fastmri - name: Install GPU related interfaces run: | - export CUDA_BIN_PATH=/usr/local/cuda-12.1/ - export PATH=/usr/local/cuda-12.1/bin/:${PATH} - export LD_LIBRARY_PATH=/usr/local/cuda-12.1/lib64/:${LD_LIBRARY_PATH} + ${{ env.activate_venv }} + ${{ env.setup_cuda }} pip install cupy-cuda12x torch python -m pip install gpuNUFFT "cufinufft<2.3" sigpy scikit-image fastmri - name: Run examples shell: bash run: | + ${{ env.activate_venv }} export COVERAGE_FILE=coverage_plots pytest examples tests -k="not operators" --cov --cov-branch --cov-report=term @@ -260,12 +274,15 @@ jobs: - name: add the coverage tool shell: bash run: | + ${{ env.create_venv }} + ${{ env.activate_venv }} python -m pip install --upgrade pip python -m pip install coverage[toml] python -m pip install -e . - name: Combine coverage run: | + ${{ env.activate_venv }} export COVERAGE_FILE=coverage_combined coverage combine -a coverage_data/* @@ -277,6 +294,7 @@ jobs: - name: Reports run: | + ${{ env.activate_venv }} export COVERAGE_FILE=coverage_combined coverage xml coverage report @@ -316,20 +334,22 @@ jobs: - name: Install dependencies shell: bash -l {0} run: | + ${{ env.create_venv }} + ${{ env.activate_venv }} python -m pip install --upgrade pip python -m pip install .[doc] python -m pip install finufft - name: Install GPU related interfaces run: | - export CUDA_BIN_PATH=/usr/local/cuda-12.1/ - export PATH=/usr/local/cuda-12.1/bin/:${PATH} - export LD_LIBRARY_PATH=/usr/local/cuda-12.1/lib64/:${LD_LIBRARY_PATH} + ${{ env.activate_venv }} + ${{ env.setup_cuda }} pip install cupy-cuda12x torch python -m pip install gpuNUFFT "cufinufft<2.3" - name: Build API documentation run: | + ${{ env.activate_venv }} python -m sphinx docs docs_build - name: Display data From 67ece186c5c1b5169f31f3d317d329e8535dcb5b Mon Sep 17 00:00:00 2001 From: Pierre-antoine Comby Date: Mon, 3 Mar 2025 13:20:12 +0100 Subject: [PATCH 2/3] more isolation. --- .github/workflows/test-ci.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/test-ci.yml b/.github/workflows/test-ci.yml index d105f9884..d6ce3bd51 100644 --- a/.github/workflows/test-ci.yml +++ b/.github/workflows/test-ci.yml @@ -16,7 +16,7 @@ env: PYTHON_VERSION: "3.10" BART_VERSION: "0.8.00" ref_backend: "finufft" - create_venv: "python -m venv $RUNNER_WORKSPACE/venv/" + create_venv: "python -m venv $RUNNER_WORKSPACE/venv/ --clear" activate_venv_venv: "source $RUNNER_WORKSPACE/venv/bin/activate" setup_cuda: "export CUDA_BIN_PATH=/usr/local/cuda-12.1/;export PATH=/usr/local/cuda-12.1/bin/:${PATH};export LD_LIBRARY_PATH=/usr/local/cuda-12.1/lib64/:${LD_LIBRARY_PATH}" @@ -104,7 +104,7 @@ jobs: run: | ${{env.activate_venv}} export COVERAGE_FILE=coverage_${{ matrix.backend }} - pytest -k='operators' --backend ${{ matrix.backend }} --ref ${{ env.ref_backend }} --cov --disable-pytest-warnings --cov-branch --cov-report=term + python -m pytest -k='operators' --backend ${{ matrix.backend }} --ref ${{ env.ref_backend }} --cov --disable-pytest-warnings --cov-branch --cov-report=term - name: Upload coverage uses: actions/upload-artifact@v4 with: @@ -129,16 +129,16 @@ jobs: run: | ${{ env.create_venv }} ${{ env.activate_venv }} - pip install --upgrade pip wheel - pip install -e mri-nufft[test] - pip install cupy-cuda12x finufft "numpy<2.0" + python -m pip install --upgrade pip wheel + python -m pip install -e mri-nufft[test] + python -m pip install cupy-cuda12x finufft "numpy<2.0" - name: Install torch with CUDA 12.1 shell: bash if: ${{ matrix.backend != 'tensorflow'}} run: | source $RUNNER_WORKSPACE/venv/binactivate_venv - pip install torch + python -m pip install torch - name: Install backend shell: bash @@ -146,13 +146,13 @@ jobs: ${{ env.activate_venv }} ${{ env.setup_cuda }} if [[ ${{ matrix.backend }} == "torchkbnufft-gpu" ]]; then - pip install torchkbnufft + python -m pip install torchkbnufft elif [[ ${{ matrix.backend }} == "tensorflow" ]]; then - pip install tensorflow-mri==0.21.0 tensorflow-probability==0.17.0 tensorflow-io==0.27.0 matplotlib==3.7 + python -m pip install tensorflow-mri==0.21.0 tensorflow-probability==0.17.0 tensorflow-io==0.27.0 matplotlib==3.7 elif [[ ${{ matrix.backend }} == "cufinufft" ]]; then - pip install "cufinufft<2.3" + python -m pip install "cufinufft<2.3" else - pip install ${{ matrix.backend }} + python -m pip install ${{ matrix.backend }} fi - name: Run Tests From d79e284c5319b6b86c8d1199405d9a8e7084136c Mon Sep 17 00:00:00 2001 From: Pierre-antoine Comby Date: Mon, 3 Mar 2025 13:26:13 +0100 Subject: [PATCH 3/3] update ci --- .github/workflows/test-ci.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/test-ci.yml b/.github/workflows/test-ci.yml index d6ce3bd51..c4d7f04cd 100644 --- a/.github/workflows/test-ci.yml +++ b/.github/workflows/test-ci.yml @@ -16,8 +16,8 @@ env: PYTHON_VERSION: "3.10" BART_VERSION: "0.8.00" ref_backend: "finufft" - create_venv: "python -m venv $RUNNER_WORKSPACE/venv/ --clear" - activate_venv_venv: "source $RUNNER_WORKSPACE/venv/bin/activate" + create_venv: "python -m venv $RUNNER_WORKSPACE/venv" + activate_venv: "source $RUNNER_WORKSPACE/venv/bin/activate" setup_cuda: "export CUDA_BIN_PATH=/usr/local/cuda-12.1/;export PATH=/usr/local/cuda-12.1/bin/:${PATH};export LD_LIBRARY_PATH=/usr/local/cuda-12.1/lib64/:${LD_LIBRARY_PATH}" jobs: @@ -48,8 +48,8 @@ jobs: shell: bash run: | python --version - ${{env.create_venv}} - ${{env.activate_venv_venv}} + ${{ env.create_venv }} + ${{ env.activate_venv }} python -m pip install --upgrade pip python -m pip install -e .[test] @@ -57,27 +57,27 @@ jobs: if: ${{ matrix.backend == 'pynfft' || env.ref_backend == 'pynfft' }} shell: bash run: | - ${{env.activate_venv}} + ${{ env.activate_venv }} python -m pip install "pynfft2>=1.4.3" - name: Install pynufft if: ${{ matrix.backend == 'pynufft-cpu' || env.ref_backend == 'pynufft-cpu' }} run: | - ${{env.activate_venv}} + ${{ env.activate_venv }} python -m pip install pynufft - name: Install finufft if: ${{ matrix.backend == 'finufft' || env.ref_backend == 'finufft'}} shell: bash run: | - ${{env.activate_venv}} + ${{ env.activate_venv }} python -m pip install finufft - name: Install Sigpy if: ${{ matrix.backend == 'sigpy' || env.ref_backend == 'sigpy'}} shell: bash run: | - ${{env.activate_venv}} + ${{ env.activate_venv }} python -m pip install sigpy - name: Install BART @@ -95,14 +95,14 @@ jobs: - name: Install torchkbnufft-cpu if: ${{ matrix.backend == 'torchkbnufft-cpu' || env.ref_backend == 'torchkbnufft-cpu'}} run: | - ${{env.activate_venv}} + ${{ env.activate_venv }} python -m pip install torchkbnufft - name: Run Tests shell: bash run: | - ${{env.activate_venv}} + ${{ env.activate_venv }} export COVERAGE_FILE=coverage_${{ matrix.backend }} python -m pytest -k='operators' --backend ${{ matrix.backend }} --ref ${{ env.ref_backend }} --cov --disable-pytest-warnings --cov-branch --cov-report=term - name: Upload coverage @@ -130,14 +130,14 @@ jobs: ${{ env.create_venv }} ${{ env.activate_venv }} python -m pip install --upgrade pip wheel - python -m pip install -e mri-nufft[test] + python -m pip install -e .[test] python -m pip install cupy-cuda12x finufft "numpy<2.0" - name: Install torch with CUDA 12.1 shell: bash if: ${{ matrix.backend != 'tensorflow'}} run: | - source $RUNNER_WORKSPACE/venv/binactivate_venv + ${{ env.activate_venv }} python -m pip install torch - name: Install backend