1541 update coverage report config (Project-MONAI#1746)

* update coverage config

Signed-off-by: Wenqi Li <[email protected]>

* temp tests

Signed-off-by: Wenqi Li <[email protected]>

* fixes https://github.com/Project-MONAI/MONAI/runs/2083800079?check_suite_focus=true#step:5:13886

Signed-off-by: Wenqi Li <[email protected]>

* test cases matching in runner

Signed-off-by: Wenqi Li <[email protected]>

* fixes openslide tests

Signed-off-by: Wenqi Li <[email protected]>

* fixes https://github.com/Project-MONAI/MONAI/runs/2086767998?check_suite_focus=true#step:7:5955

Signed-off-by: Wenqi Li <[email protected]>

* fixes print stats

Signed-off-by: Wenqi Li <[email protected]>

* remove temp tests

Signed-off-by: Wenqi Li <[email protected]>

* remove unused

Signed-off-by: Wenqi Li <[email protected]>

* remove global logging config

Signed-off-by: Wenqi Li <[email protected]>

* omit setup.py

Signed-off-by: Wenqi Li <[email protected]>

.dockerignore

@@ -4,6 +4,9 @@ __pycache__/
 docs/
 
 .coverage
+.coverage.*
+.coverage/
+coverage.xml
 .readthedocs.yml
 *.md
 *.toml

.gitignore

@@ -40,6 +40,7 @@ htmlcov/
 .tox/
 .coverage
 .coverage.*
+.coverage/
 .cache
 nosetests.xml
 coverage.xml
@@ -124,6 +125,7 @@ temp/
 # temporary testing data MedNIST
 tests/testing_data/MedNIST*
 tests/testing_data/*Hippocampus*
+tests/testing_data/CMU-1.tiff
 
 # clang format tool
 .clang-format-bin/

monai/transforms/utility/array.py

@@ -14,6 +14,7 @@
 """
 
 import logging
+import sys
 import time
 from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Sequence, Tuple, Union
 
@@ -409,6 +410,7 @@ def __init__(
             additional_info: user can define callable function to extract additional info from input data.
             logger_handler: add additional handler to output data: save to file, etc.
                 add existing python logging handlers: https://docs.python.org/3/library/logging.handlers.html
+                the handler should have a logging level of at least `INFO`.
 
         Raises:
             TypeError: When ``additional_info`` is not an ``Optional[Callable]``.
@@ -424,8 +426,11 @@ def __init__(
             raise TypeError(f"additional_info must be None or callable but is {type(additional_info).__name__}.")
         self.additional_info = additional_info
         self.output: Optional[str] = None
-        logging.basicConfig(level=logging.NOTSET)
         self._logger = logging.getLogger("DataStats")
+        self._logger.setLevel(logging.INFO)
+        console = logging.StreamHandler(sys.stdout)  # always stdout
+        console.setLevel(logging.INFO)
+        self._logger.addHandler(console)
         if logger_handler is not None:
             self._logger.addHandler(logger_handler)
 
@@ -459,7 +464,7 @@ def __call__(
             lines.append(f"Additional info: {additional_info(img)}")
         separator = "\n"
         self.output = f"{separator.join(lines)}"
-        self._logger.debug(self.output)
+        self._logger.info(self.output)
 
         return img
 

monai/transforms/utility/dictionary.py

@@ -532,6 +532,7 @@ def __init__(
                 corresponds to a key in ``keys``.
             logger_handler: add additional handler to output data: save to file, etc.
                 add existing python logging handlers: https://docs.python.org/3/library/logging.handlers.html
+                the handler should have a logging level of at least `INFO`.
             allow_missing_keys: don't raise exception if key is missing.
 
         """

requirements-min.txt

@@ -1,5 +1,5 @@
 # Requirements for minimal tests
 -r requirements.txt
 setuptools>=50.3.0
-coverage
+coverage>=5.5
 parameterized

runtests.sh

@@ -138,6 +138,9 @@ function clang_format {
 }
 
 function clean_py {
+    # remove coverage history
+    ${cmdPrefix}${PY_EXE} -m coverage erase
+
     # uninstall the development package
     echo "Uninstalling MONAI development files..."
     ${cmdPrefix}${PY_EXE} setup.py develop --user --uninstall
@@ -149,7 +152,7 @@ function clean_py {
     find ${TO_CLEAN}/monai -type f -name "*.py[co]" -delete
     find ${TO_CLEAN}/monai -type f -name "*.so" -delete
     find ${TO_CLEAN}/monai -type d -name "__pycache__" -delete
-    find ${TO_CLEAN} -maxdepth 1 -type f -name ".coverage" -delete
+    find ${TO_CLEAN} -maxdepth 1 -type f -name ".coverage.*" -delete
 
     find ${TO_CLEAN} -depth -maxdepth 1 -type d -name ".eggs" -exec rm -r "{}" +
     find ${TO_CLEAN} -depth -maxdepth 1 -type d -name "monai.egg-info" -exec rm -r "{}" +
@@ -496,12 +499,11 @@ then
     export QUICKTEST=True
 fi
 
-# set command and clear previous coverage data
+# set coverage command
 if [ $doCoverage = true ]
 then
     echo "${separator}${blue}coverage${noColor}"
-    cmd="${PY_EXE} -m coverage run -a --source ."
-    ${cmdPrefix}${PY_EXE} -m coverage erase
+    cmd="${PY_EXE} -m coverage run --append"
 fi
 
 # # download test data if needed
@@ -514,7 +516,7 @@ if [ $doUnitTests = true ]
 then
     echo "${separator}${blue}unittests${noColor}"
     torch_validate
-    ${cmdPrefix}${cmd} ./tests/runner.py -p "test_[!integration]*py"
+    ${cmdPrefix}${cmd} ./tests/runner.py -p "test_((?!integration).)"
 fi
 
 # network training/inference/eval integration tests
@@ -540,5 +542,6 @@ fi
 if [ $doCoverage = true ]
 then
     echo "${separator}${blue}coverage${noColor}"
-    ${cmdPrefix}${PY_EXE} -m coverage report --skip-covered -m
+    ${cmdPrefix}${PY_EXE} -m coverage combine --append .coverage/
+    ${cmdPrefix}${PY_EXE} -m coverage report
 fi

setup.cfg

@@ -55,19 +55,19 @@ lmdb =
     lmdb
 psutil =
     psutil
-openslide = 
+openslide =
     openslide-python==1.1.2
 
 [flake8]
 select = B,C,E,F,N,P,T4,W,B9
-max-line-length = 120
+max_line_length = 120
 # C408 ignored because we like the dict keyword argument syntax
 # E501 is not flexible enough, we're using B950 instead
 ignore =
     E203,E305,E402,E501,E721,E741,F821,F841,F999,W503,W504,C408,E302,W291,E303,
     # N812 lowercase 'torch.nn.functional' imported as non lowercase 'F'
     N812
-per-file-ignores = __init__.py: F401
+per_file_ignores = __init__.py: F401
 exclude = *.pyi,.git,.eggs,monai/_version.py,versioneer.py,venv,.venv,_version.py
 
 [isort]
@@ -148,3 +148,21 @@ precise_return = True
 protocols = True
 # Experimental: Only load submodules that are explicitly imported.
 strict_import = False
+
+[coverage:run]
+concurrency = multiprocessing
+source = .
+data_file = .coverage/.coverage
+omit = setup.py
+
+[coverage:report]
+exclude_lines =
+    pragma: no cover
+    # Don't complain if tests don't hit code:
+    raise NotImplementedError
+    if __name__ == .__main__.:
+show_missing = True
+skip_covered = True
+
+[coverage:xml]
+output = coverage.xml

tests/runner.py

@@ -10,8 +10,10 @@
 # limitations under the License.
 
 import argparse
+import glob
 import inspect
 import os
+import re
 import sys
 import time
 import unittest
@@ -62,7 +64,7 @@ def print_results(results, discovery_time, thresh, status):
     print("Remember to check above times for any errors!")
 
 
-def parse_args(default_pattern):
+def parse_args():
     parser = argparse.ArgumentParser(description="Runner for MONAI unittests with timing.")
     parser.add_argument(
         "-s", action="store", dest="path", default=".", help="Directory to start discovery (default: '%(default)s')"
@@ -71,7 +73,7 @@ def parse_args(default_pattern):
         "-p",
         action="store",
         dest="pattern",
-        default=default_pattern,
+        default="test_*.py",
         help="Pattern to match tests (default: '%(default)s')",
     )
     parser.add_argument(
@@ -111,22 +113,26 @@ def get_default_pattern(loader):
 
 if __name__ == "__main__":
 
-    loader = unittest.TestLoader()
-    default_pattern = get_default_pattern(loader)
-
     # Parse input arguments
-    args = parse_args(default_pattern)
+    args = parse_args()
 
     # If quick is desired, set environment variable
     if args.quick:
         os.environ["QUICKTEST"] = "True"
 
     # Get all test names (optionally from some path with some pattern)
     with PerfContext() as pc:
-        tests = loader.discover(args.path, args.pattern)
+        # the files are searched from `tests/` folder, starting with `test_`
+        files = glob.glob(os.path.join(os.path.dirname(__file__), "test_*.py"))
+        cases = []
+        for test_module in {os.path.basename(f)[:-3] for f in files}:
+            if re.match(args.pattern, test_module):
+                cases.append(f"tests.{test_module}")
+            else:
+                print(f"monai test runner: excluding tests.{test_module}")
+        tests = unittest.TestLoader().loadTestsFromNames(cases)
     discovery_time = pc.total_time
-    print(f"time to discover tests: {discovery_time}s")
-    print(tests)
+    print(f"time to discover tests: {discovery_time}s, total cases: {tests.countTestCases()}.")
 
     test_runner = unittest.runner.TextTestRunner(
         resultclass=TimeLoggingTestResult, verbosity=args.verbosity, failfast=args.failfast

tests/test_affine.py

@@ -80,10 +80,7 @@ def test_affine(self, input_param, input_data, expected_val):
         g = Affine(**input_param)
         result = g(**input_data)
         self.assertEqual(isinstance(result, torch.Tensor), isinstance(expected_val, torch.Tensor))
-        if isinstance(result, torch.Tensor):
-            np.testing.assert_allclose(result.cpu().numpy(), expected_val.cpu().numpy(), rtol=1e-4, atol=1e-4)
-        else:
-            np.testing.assert_allclose(result, expected_val, rtol=1e-4, atol=1e-4)
+        np.testing.assert_allclose(result, expected_val, rtol=1e-4, atol=1e-4)
 
 
 if __name__ == "__main__":

tests/test_affined.py

@@ -94,10 +94,7 @@ def test_affine(self, input_param, input_data, expected_val):
         g = Affined(**input_param)
         result = g(input_data)["img"]
         self.assertEqual(isinstance(result, torch.Tensor), isinstance(expected_val, torch.Tensor))
-        if isinstance(result, torch.Tensor):
-            np.testing.assert_allclose(result.cpu().numpy(), expected_val.cpu().numpy(), rtol=1e-4, atol=1e-4)
-        else:
-            np.testing.assert_allclose(result, expected_val, rtol=1e-4, atol=1e-4)
+        np.testing.assert_allclose(result, expected_val, rtol=1e-4, atol=1e-4)
 
 
 if __name__ == "__main__":

tests/test_data_stats.py

@@ -119,6 +119,7 @@ def test_file(self, input_data, expected_print):
         with tempfile.TemporaryDirectory() as tempdir:
             filename = os.path.join(tempdir, "test_data_stats.log")
             handler = logging.FileHandler(filename, mode="w")
+            handler.setLevel(logging.INFO)
             input_param = {
                 "prefix": "test data",
                 "data_shape": True,
@@ -129,8 +130,9 @@ def test_file(self, input_data, expected_print):
             }
             transform = DataStats(**input_param)
             _ = transform(input_data)
-            handler.stream.close()
-            transform._logger.removeHandler(handler)
+            for h in transform._logger.handlers[:]:
+                h.close()
+                transform._logger.removeHandler(h)
             with open(filename, "r") as f:
                 content = f.read()
             self.assertEqual(content, expected_print)

tests/test_data_statsd.py

@@ -132,6 +132,7 @@ def test_file(self, input_data, expected_print):
         with tempfile.TemporaryDirectory() as tempdir:
             filename = os.path.join(tempdir, "test_stats.log")
             handler = logging.FileHandler(filename, mode="w")
+            handler.setLevel(logging.INFO)
             input_param = {
                 "keys": "img",
                 "prefix": "test data",
@@ -143,8 +144,10 @@ def test_file(self, input_data, expected_print):
             }
             transform = DataStatsd(**input_param)
             _ = transform(input_data)
-            handler.stream.close()
-            transform.printer._logger.removeHandler(handler)
+            for h in transform.printer._logger.handlers[:]:
+                h.close()
+                transform.printer._logger.removeHandler(h)
+            del handler
             with open(filename, "r") as f:
                 content = f.read()
             self.assertEqual(content, expected_print)