249 add weekly tests (#295)

Fixes #249 .

### Description
This PR adds the weekly ci test for existing bundles. It will download
all bundles (latest version) from release, and then do the ci test
(cpu).

### Status
**Ready**

### Please ensure all the checkboxes:
<!--- Put an `x` in all the boxes that apply, and remove the not
applicable items -->
- [x] Codeformat tests passed locally by running `./runtests.sh
--codeformat`.
- [ ] In-line docstrings updated.
- [ ] Update `version` and `changelog` in `metadata.json` if changing an
existing bundle.
- [ ] Please ensure the naming rules in config files meet our
requirements (please refer to: `CONTRIBUTING.md`).
- [ ] Ensure versions of packages such as `monai`, `pytorch` and `numpy`
are correct in `metadata.json`.
- [ ] Descriptions should be consistent with the content, such as
`eval_metrics` of the provided weights and TorchScript modules.
- [ ] Files larger than 25MB are excluded and replaced by providing
download links in `large_file.yml`.
- [ ] Avoid using path that contains personal information within config
files (such as use `/home/your_name/` for `"bundle_root"`).

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

.github/workflows/weekly-tests-cpu.yml

@@ -0,0 +1,72 @@
+name: weekly-tests-cpu
+
+on:
+  # weekly tests for existing bundles in release
+  schedule:
+  - cron: "0 5 * * 0"  # 5:00, every Sunday
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+jobs:
+  get-list:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.8
+    - name: cache weekly timestamp
+      id: pip-cache
+      run: |
+        echo "datew=$(date '+%Y-%V')" >> $GITHUB_OUTPUT
+    - name: cache for pip
+      uses: actions/cache@v2
+      id: cache
+      with:
+        path: ~/.cache/pip
+        key: ${{ runner.os }}-pip-${{ steps.pip-cache.outputs.datew }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip wheel
+        python -m pip install -r requirements-dev.txt
+    - name: get bundle list
+      id: get-bundle-list
+      run: |
+        # clean up temporary files
+        $(pwd)/runtests.sh --clean
+        echo "list=$(python $(pwd)/ci/get_bundle_list.py --models_path $(pwd)/models/)" >> $GITHUB_OUTPUT
+      shell: bash
+    outputs:
+      list: ${{ steps.get-bundle-list.outputs.list }}
+  test-cpu:
+    needs: get-list
+    strategy:
+      matrix:
+        bundle: ${{ fromJson(needs.get-list.outputs.list) }}
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.8
+    - name: cache weekly timestamp
+      id: pip-cache
+      run: |
+        echo "::set-output name=datew::$(date '+%Y-%V')"
+    - name: cache for pip
+      uses: actions/cache@v2
+      id: cache
+      with:
+        path: ~/.cache/pip
+        key: ${{ runner.os }}-pip-${{ steps.pip-cache.outputs.datew }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip wheel
+    - name: check
+      run: |
+        # clean up temporary files
+        $(pwd)/runtests.sh --clean
+        bash $(pwd)/ci/run_regular_tests_cpu.sh ${{ matrix.bundle }}
+      shell: bash

ci/download_latest_bundle.py

@@ -0,0 +1,36 @@
+# Copyright (c) MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import argparse
+import os
+
+from monai.bundle import download
+from utils import get_latest_version
+
+
+def main(bundle_name: str, models_path: str, download_path: str):
+
+    model_info_path = os.path.join(models_path, "model_info.json")
+    version = get_latest_version(bundle_name=bundle_name, model_info_path=model_info_path)
+    download(name=bundle_name, source="github", version=version, bundle_dir=download_path)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="")
+    parser.add_argument("-b", "--b", type=str, help="bundle name.")
+    parser.add_argument("-models_path", "--models_path", type=str, help="models path.")
+    parser.add_argument("-p", "--p", type=str, help="download path.")
+    args = parser.parse_args()
+    bundle_name = args.b
+    models_path = args.models_path
+    download_path = args.p
+    main(bundle_name, models_path, download_path)

ci/get_bundle_list.py

@@ -0,0 +1,30 @@
+# Copyright (c) MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import argparse
+
+from utils import get_sub_folders
+
+
+def main(models_path):
+
+    bundle_list = get_sub_folders(root_dir=models_path)
+
+    print(bundle_list)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="")
+    parser.add_argument("-models_path", "--models_path", type=str, help="models path.")
+    args = parser.parse_args()
+    models_path = args.models_path
+    main(models_path)

ci/get_bundle_requirements.py

@@ -16,9 +16,8 @@
 from utils import get_json_dict
 
 
-def get_requirements(bundle):
+def get_requirements(bundle, models_path):
 
-    models_path = "models"
     bundle_path = os.path.join(models_path, bundle)
     meta_file_path = os.path.join(bundle_path, "configs/metadata.json")
     if os.path.exists(meta_file_path):
@@ -49,6 +48,8 @@ def get_requirements(bundle):
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="")
     parser.add_argument("-b", "--b", type=str, help="bundle name.")
+    parser.add_argument("-p", "--p", type=str, default="models", help="models path.")
     args = parser.parse_args()
     bundle = args.b
-    get_requirements(bundle)
+    models_path = args.p
+    get_requirements(bundle, models_path)

ci/prepare_schema.py

@@ -15,16 +15,17 @@
 from utils import prepare_schema
 
 
-def main(bundle_list):
+def main(bundle_list, models_path):
 
-    root_path = "models"
-    prepare_schema(bundle_list, root_path=root_path)
+    prepare_schema(bundle_list, root_path=models_path)
 
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="")
     parser.add_argument("-l", "--l", type=str, help="bundle list.")
+    parser.add_argument("-p", "--p", type=str, default="models", help="models path.")
     args = parser.parse_args()
     bundle_list = args.l.split(" ")
     bundle_list = [f for f in bundle_list if f != ""]
-    main(bundle_list)
+    models_path = args.p
+    main(bundle_list, models_path)

ci/run_regular_tests_cpu.sh

@@ -0,0 +1,56 @@
+#!/bin/bash
+#
+# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Argument(s):
+#   bundle:   bundle to be tested
+
+set -ex
+bundle=""
+
+if [[ $# -eq 1 ]]; then
+    bundle=$1
+
+elif [[ $# -gt 1 ]]; then
+    echo "ERROR: too many parameters are provided"
+    exit 1
+fi
+
+verify_release_bundle() {
+    echo 'Run verify bundle...'
+    # get all bundles
+    download_path="download"
+    pip install -r requirements-dev.txt
+    # download bundle from releases
+    python $(pwd)/ci/download_latest_bundle.py --b "$bundle" --models_path $(pwd)/models --p "$download_path"
+    # get required libraries according to the bundle's metadata file
+    requirements=$(python $(pwd)/ci/get_bundle_requirements.py --b "$bundle" --p "$download_path")
+    if [ ! -z "$requirements" ]; then
+        echo "install required libraries for bundle: $bundle"
+        pip install -r "$requirements"
+    fi
+    # verify bundle
+    python $(pwd)/ci/verify_bundle.py -b "$bundle" -p "$download_path" -m "regular"  # regular tests on cpu
+}
+
+
+case $bundle in
+
+    *)
+        echo "Check bundle: $bundle"
+        verify_release_bundle
+        ;;
+esac

ci/utils.py

@@ -110,6 +110,16 @@ def save_model_info(model_info_dict, model_info_path: str):
         json.dump(model_info_dict, f)
 
 
+def get_latest_version(bundle_name: str, model_info_path: str):
+    model_info_dict = get_json_dict(model_info_path)
+    versions = []
+    for k in model_info_dict.keys():
+        if bundle_name in k:
+            versions.append(k.split(f"{bundle_name}_v")[1])
+
+    return sorted(versions)[-1]
+
+
 def push_new_model_info_branch(model_info_path: str):
     email = os.environ["email"]
     username = os.environ["username"]

ci/verify_bundle.py

@@ -285,9 +285,8 @@ def verify_torchscript(bundle_path: str, net_id: str, config_file: str):
         print("Provided TorchScript module is verified correctly.")
 
 
-def verify(bundle, mode="full"):
+def verify(bundle, models_path="models", mode="full"):
 
-    models_path = "models"
     print(f"start verifying {bundle}:")
     # add bundle path to ensure custom code can be used
     sys.path = [os.path.join(models_path, bundle)] + sys.path
@@ -297,18 +296,19 @@ def verify(bundle, mode="full"):
     # verify bundle keys
     verify_bundle_keys(models_path, bundle)
     print("keys are verified correctly.")
-    # verify version, changelog
-    verify_version_changes(models_path, bundle)
-    print("version and changelog are verified correctly.")
+    if mode != "regular":
+        # verify version, changelog
+        verify_version_changes(models_path, bundle)
+        print("version and changelog are verified correctly.")
     # verify metadata format and data
     bundle_path = os.path.join(models_path, bundle)
     verify_metadata_format(bundle_path)
     print("metadata format is verified correctly.")
 
-    if mode == "min":
+    if mode in ["min", "regular"]:
         return
 
-    # The following are optional tests
+    # The following are optional tests and require GPU
     net_id, inference_file_name = "network_def", _find_bundle_file(os.path.join(bundle_path, "configs"), "inference")
     config_file = os.path.join("configs", inference_file_name)
 
@@ -318,7 +318,6 @@ def verify(bundle, mode="full"):
         verify_data_shape(bundle_path, net_id, config_file)
         print("data shape is verified correctly.")
 
-    # The following tests require to use GPU
     if bundle in exclude_verify_torchscript_list:
         print(f"bundle: {bundle} does not support torchscript, skip verifying.")
     else:
@@ -328,8 +327,10 @@ def verify(bundle, mode="full"):
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="")
     parser.add_argument("-b", "--b", type=str, help="bundle name.")
+    parser.add_argument("-p", "--p", type=str, default="models", help="models path.")
     parser.add_argument("-m", "--mode", type=str, default="full", help="verify bundle mode (full/min).")
     args = parser.parse_args()
     bundle = args.b
+    models_path = args.p
     mode = args.mode
-    verify(bundle, mode)
+    verify(bundle, models_path, mode)

requirements.txt

@@ -1 +1 @@
-monai>=0.9.0
+monai>=1.0.1