1236 integration test memory (Project-MONAI#1738)

* addresses testing instance memory errors Signed-off-by: Wenqi Li <[email protected]>
gagandaroach · Mar 11, 2021 · 6d3bf2f · 6d3bf2f
1 parent c3e5a69
commit 6d3bf2f
Show file tree

Hide file tree

Showing 8 changed files with 66 additions and 35 deletions.
diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
@@ -11,7 +11,7 @@ A few sentences describing the changes proposed in this pull request.
 - [x] Non-breaking change (fix or new feature that would not break existing functionality).
 - [ ] Breaking change (fix or new feature that would cause existing functionality to change).
 - [ ] New tests added to cover the changes.
-- [ ] Integration tests passed locally by running `./runtests.sh --codeformat --coverage`.
-- [ ] Quick tests passed locally by running `./runtests.sh --quick`.
+- [ ] Integration tests passed locally by running `./runtests.sh -f -u --net --coverage`.
+- [ ] Quick tests passed locally by running `./runtests.sh --quick --unittests`.
 - [ ] In-line docstrings updated.
 - [ ] Documentation updated, tested `make html` command in the `docs/` folder.
diff --git a/.github/workflows/cron.yml b/.github/workflows/cron.yml
@@ -40,9 +40,16 @@ jobs:
         nvidia-smi
         export CUDA_VISIBLE_DEVICES=$(python -m tests.utils)
         echo $CUDA_VISIBLE_DEVICES
+        stop_time=$((LAUNCH_DELAY + $(date +%s)))
+        while [ $(date +%s) -lt $stop_time ]; do
+          python -c 'import torch; torch.rand(5, 3, device=torch.device("cuda:0"))';
+        done
         python -c "import torch; print(torch.__version__); print('{} of GPUs available'.format(torch.cuda.device_count()))"
-        python -c 'import torch; print(torch.rand(5,3, device=torch.device("cuda:0")))'
-        BUILD_MONAI=1 ./runtests.sh --coverage
+        python -c 'import torch; print(torch.rand(5, 3, device=torch.device("cuda:0")))'
+        BUILD_MONAI=1 ./runtests.sh --coverage --unittests  # unit tests with coverage report
+        export CUDA_VISIBLE_DEVICES=$(python -m tests.utils)
+        echo $CUDA_VISIBLE_DEVICES
+        BUILD_MONAI=1 ./runtests.sh --coverage --net  # integration tests with coverage report
         coverage xml
     - name: Upload coverage
       uses: codecov/codecov-action@v1
@@ -76,9 +83,16 @@ jobs:
         nvidia-smi
         export CUDA_VISIBLE_DEVICES=$(python -m tests.utils)
         echo $CUDA_VISIBLE_DEVICES
+        stop_time=$((LAUNCH_DELAY + $(date +%s)))
+        while [ $(date +%s) -lt $stop_time ]; do
+          python -c 'import torch; torch.rand(5, 3, device=torch.device("cuda:0"))';
+        done
         python -c "import torch; print(torch.__version__); print('{} of GPUs available'.format(torch.cuda.device_count()))"
-        python -c 'import torch; print(torch.rand(5,3, device=torch.device("cuda:0")))'
-        BUILD_MONAI=1 ./runtests.sh --coverage
+        python -c 'import torch; print(torch.rand(5, 3, device=torch.device("cuda:0")))'
+        BUILD_MONAI=1 ./runtests.sh --coverage --unittests  # unit tests with coverage report
+        export CUDA_VISIBLE_DEVICES=$(python -m tests.utils)
+        echo $CUDA_VISIBLE_DEVICES
+        BUILD_MONAI=1 ./runtests.sh --coverage --net  # integration tests with coverage report
         coverage xml
     - name: Upload coverage
       uses: codecov/codecov-action@v1
@@ -104,7 +118,10 @@ jobs:
         python -c "import torch; print(torch.__version__); print('{} of GPUs available'.format(torch.cuda.device_count()))"
         python -c 'import torch; print(torch.rand(5,3, device=torch.device("cuda:0")))'
         ngc --version
-        BUILD_MONAI=1 ./runtests.sh --coverage --pytype
+        BUILD_MONAI=1 ./runtests.sh --coverage --pytype --unittests  # unit tests with pytype checks, coverage report
+        export CUDA_VISIBLE_DEVICES=$(python -m tests.utils)
+        echo $CUDA_VISIBLE_DEVICES
+        BUILD_MONAI=1 ./runtests.sh --coverage --net  # integration tests with coverage report
         coverage xml
     - name: Upload coverage
       uses: codecov/codecov-action@v1

diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
@@ -44,7 +44,7 @@ jobs:
         echo $CUDA_VISIBLE_DEVICES
         python -c "import torch; print(torch.__version__); print('{} of GPUs available'.format(torch.cuda.device_count()))"
         python -c 'import torch; print(torch.rand(5,3, device=torch.device("cuda:0")))'
-        BUILD_MONAI=1 ./runtests.sh --net
+        BUILD_MONAI=1 ./runtests.sh --unittests --net
     - name: Add reaction
       uses: peter-evans/create-or-update-comment@v1
       with:

diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml
@@ -39,7 +39,7 @@ jobs:
         # clean up temporary files
         $(pwd)/runtests.sh --clean
         # Git hub actions have 2 cores, so parallize pytype
-        $(pwd)/runtests.sh --nounittests --codeformat -j 2
+        $(pwd)/runtests.sh --codeformat -j 2
 
   quick-py3:  # full dependencies installed
     runs-on: ${{ matrix.os }}
@@ -241,7 +241,7 @@ jobs:
         python -c "import torch; print(torch.__version__); print('{} of GPUs available'.format(torch.cuda.device_count()))"
         python -c 'import torch; print(torch.rand(5, 3, device=torch.device("cuda:0")))'
         python -c "import monai; monai.config.print_config()"
-        BUILD_MONAI=1 ./runtests.sh --quick
+        BUILD_MONAI=1 ./runtests.sh --quick --unittests
         if [ ${{ matrix.environment }} == "PT18+CUDA112" ]; then
           # test the clang-format tool downloading once
           coverage run -m tests.clang_format_utils

diff --git a/.github/workflows/setupapp.yml b/.github/workflows/setupapp.yml
@@ -41,12 +41,22 @@ jobs:
     - name: Run unit tests report coverage
       run: |
         python -m pip list
+        export LAUNCH_DELAY=$[ $RANDOM % 16 * 60 ]
+        echo "Sleep $LAUNCH_DELAY"
+        sleep $LAUNCH_DELAY
         nvidia-smi
         export CUDA_VISIBLE_DEVICES=$(python -m tests.utils)
         echo $CUDA_VISIBLE_DEVICES
+        stop_time=$((LAUNCH_DELAY + $(date +%s)))
+        while [ $(date +%s) -lt $stop_time ]; do
+          python -c 'import torch; torch.rand(5, 3, device=torch.device("cuda:0"))';
+        done
         python -c "import torch; print(torch.__version__); print('{} of GPUs available'.format(torch.cuda.device_count()))"
-        python -c 'import torch; print(torch.rand(5,3, device=torch.device("cuda:0")))'
-        BUILD_MONAI=1 ./runtests.sh --coverage
+        python -c 'import torch; print(torch.rand(5, 3, device=torch.device("cuda:0")))'
+        BUILD_MONAI=1 ./runtests.sh --coverage --unittests  # unit tests with coverage report
+        export CUDA_VISIBLE_DEVICES=$(python -m tests.utils)
+        echo $CUDA_VISIBLE_DEVICES
+        BUILD_MONAI=1 ./runtests.sh --coverage --net  # integration tests with coverage report
         coverage xml
     - name: Upload coverage
       uses: codecov/codecov-action@v1
@@ -88,7 +98,7 @@ jobs:
       run: |
         python -m pip list
         python -c 'import torch; print(torch.__version__); print(torch.rand(5,3))'
-        BUILD_MONAI=1 ./runtests.sh --quick
+        BUILD_MONAI=1 ./runtests.sh --quick --unittests
         coverage xml
     - name: Upload coverage
       uses: codecov/codecov-action@v1

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -52,10 +52,10 @@ Before submitting a pull request, we recommend that all linting should pass, by
 
 ```bash
 pip install -U -r requirements-dev.txt  # install the latest tools
-./runtests.sh --codeformat --nounittests  # runs the linting tools only
+./runtests.sh --codeformat  # runs the linting tools
 
 # try to fix the coding style errors automatically
-./runtests.sh --autofix --nounittests
+./runtests.sh --autofix
 ```
 
 License information: all source code files should start with this paragraph:
@@ -98,12 +98,12 @@ Before submitting a pull request, we recommend that all linting and unit tests
 should pass, by running the following command locally:
 
 ```bash
-./runtests.sh --codeformat --coverage
+./runtests.sh --codeformat --coverage --unittests
 ```
 or (for new features that would not break existing functionality):
 
 ```bash
-./runtests.sh --quick
+./runtests.sh --quick --unittests
 ```
 
 It is recommended that the new test `test_[module_name].py` is constructed by using only

diff --git a/runtests.sh b/runtests.sh
@@ -36,9 +36,7 @@ doQuickTests=false
 doNetTests=false
 doDryRun=false
 doZooTests=false
-
-doUnitTests=true
-
+doUnitTests=false
 doBlackFormat=false
 doBlackFix=false
 doIsortFormat=false
@@ -55,16 +53,17 @@ PY_EXE=${MONAI_PY_EXE:-$(which python)}
 
 function print_usage {
     echo "runtests.sh [--codeformat] [--autofix] [--black] [--isort] [--flake8] [--clangformat] [--pytype] [--mypy]"
-    echo "            [--nounittests] [--coverage] [--quick] [--net] [--dryrun] [-j number] [--clean] [--help] [--version]"
+    echo "            [--unittests] [--coverage] [--quick] [--net] [--dryrun] [-j number] [--clean] [--help] [--version]"
     echo ""
     echo "MONAI unit testing utilities."
     echo ""
     echo "Examples:"
-    echo "./runtests.sh --codeformat --coverage     # run full tests (${green}recommended before making pull requests${noColor})."
-    echo "./runtests.sh --codeformat --nounittests  # run coding style and static type checking."
-    echo "./runtests.sh --quick                     # run minimal unit tests, for quick verification during code developments."
-    echo "./runtests.sh --autofix --nounittests     # run automatic code formatting using \"isort\" and \"black\"."
-    echo "./runtests.sh --clean                     # clean up temporary files and run \"${PY_EXE} setup.py develop --uninstall\"."
+    echo "./runtests.sh -f -u --net --coverage  # run style checks, full tests, print code coverage (${green}recommended for pull requests${noColor})."
+    echo "./runtests.sh -f -u                   # run style checks and unit tests."
+    echo "./runtests.sh -f                      # run coding style and static type checking."
+    echo "./runtests.sh --quick --unittests     # run minimal unit tests, for quick verification during code developments."
+    echo "./runtests.sh --autofix               # run automatic code formatting using \"isort\" and \"black\"."
+    echo "./runtests.sh --clean                 # clean up temporary files and run \"${PY_EXE} setup.py develop --uninstall\"."
     echo ""
     echo "Code style check options:"
     echo "    --black           : perform \"black\" code format checks"
@@ -79,11 +78,11 @@ function print_usage {
     echo "    -j, --jobs        : number of parallel jobs to run \"pytype\" (default $NUM_PARALLEL)"
     echo ""
     echo "MONAI unit testing options:"
-    echo "    --nounittests     : skip doing unit testing (i.e. only format lint testers)"
-    echo "    --coverage        : peforms coverage analysis of code for tests run"
-    echo "    -q, --quick       : disable long running tests"
-    echo "    --net             : perform training/inference/eval integration testing"
-    echo "    --list_tests      : list tests and exit"
+    echo "    -u, --unittests   : perform unit testing"
+    echo "    --coverage        : report testing code coverage, to be used with \"--net\", \"--unittests\""
+    echo "    -q, --quick       : skip long running unit tests and integration tests"
+    echo "    --net             : perform integration testing"
+    echo "    --list_tests      : list unit tests and exit"
     echo ""
     echo "Misc. options:"
     echo "    --dryrun          : display the commands to the screen without running"
@@ -92,7 +91,7 @@ function print_usage {
     echo "    -h, --help        : show this help message and exit"
     echo "    -v, --version     : show MONAI and system version information and exit"
     echo ""
-    echo "${separator}For bug reports, questions, and discussions, please file an issue at:"
+    echo "${separator}For bug reports and feature requests, please file an issue at:"
     echo "    https://github.com/Project-MONAI/MONAI/issues/new/choose"
     echo ""
     echo "To choose an alternative python executable, set the environmental variable, \"MONAI_PY_EXE\"."
@@ -220,8 +219,8 @@ do
         --dryrun)
             doDryRun=true
         ;;
-        --nou*)  # allow --nounittest | --nounittests | --nounittesting  etc.
-            doUnitTests=false
+        -u|--u*)  # allow --unittest | --unittests | --unittesting  etc.
+            doUnitTests=true
         ;;
         -f|--codeformat)
             doBlackFormat=true
@@ -268,6 +267,10 @@ do
             print_version
             exit 1
         ;;
+        --nou*)  # allow --nounittest | --nounittests | --nounittesting  etc.
+            print_error_msg "nounittest option is deprecated, no unit tests is the default setting"
+            print_usage
+        ;;
         *)
             print_error_msg "Incorrect commandline provided, invalid key: $key"
             print_usage
@@ -511,7 +514,7 @@ if [ $doUnitTests = true ]
 then
     echo "${separator}${blue}unittests${noColor}"
     torch_validate
-    ${cmdPrefix}${cmd} ./tests/runner.py
+    ${cmdPrefix}${cmd} ./tests/runner.py -p "test_[!integration]*py"
 fi
 
 # network training/inference/eval integration tests

diff --git a/tests/runner.py b/tests/runner.py
@@ -126,6 +126,7 @@ def get_default_pattern(loader):
         tests = loader.discover(args.path, args.pattern)
     discovery_time = pc.total_time
     print(f"time to discover tests: {discovery_time}s")
+    print(tests)
 
     test_runner = unittest.runner.TextTestRunner(
         resultclass=TimeLoggingTestResult, verbosity=args.verbosity, failfast=args.failfast