Updates to CI tests to run regression tests (#354)

.github/workflows/ci_build_scm_gnu9.4.yml → .github/workflows/ci_run_scm_rts.yml

@@ -1,9 +1,9 @@
-name: CI test to build SCM
+name: CI test to build and run SCM regression tests
 
-on: [push, pull_request]
+on: [pull_request]
 
 jobs:
-  build-linux:
+  run_scm_rts:
 
     # The type of runner that the job will run on
     runs-on: ubuntu-20.04
@@ -21,8 +21,8 @@ jobs:
       SCM_ROOT:   /home/runner/work/ccpp-scm/ccpp-scm
       suites:     SCM_GFS_v15p2,SCM_GFS_v16,SCM_GFS_v17_p8,SCM_HRRR,SCM_RRFS_v1beta,SCM_RAP,SCM_WoFS_v0
       suites_ps:  SCM_GFS_v15p2_ps,SCM_GFS_v16_ps,SCM_GFS_v17_p8_ps,SCM_HRRR_ps,SCM_RRFS_v1beta_ps,SCM_RAP_ps,SCM_WoFS_v0_ps
-      RUN_DIR:    ${SCM_ROOT}/scm/run_${{matrix.fortran-compiler}}_${{matrix.build-type}}
-      BIN_DIR:    ${SCM_ROOT}/scm/bin_${{matrix.fortran-compiler}}_${{matrix.build-type}}
+      dir_rt:     /home/runner/work/ccpp-scm/ccpp-scm/test/artifact-${{matrix.build-type}}
+      dir_bl:     /home/runner/work/ccpp-scm/ccpp-scm/test/BL-${{matrix.build-type}}
 
     # Workflow steps
     steps:
@@ -172,5 +172,30 @@ jobs:
     - name: Run SCM RTs
       run: |
         cd ${SCM_ROOT}/scm/bin
-        mkdir ${RUN_DIR} && mkdir ${BIN_DIR}
-        ./run_scm.py --file /home/runner/work/ccpp-scm/ccpp-scm/test/rt_test_cases.py --runtime_mult 0.1
+        ./run_scm.py --file /home/runner/work/ccpp-scm/ccpp-scm/test/rt_test_cases.py --runtime_mult 0.1
+
+    - name: Gather SCM RT output
+      run: |
+        cd ${SCM_ROOT}/test
+        mkdir ${dir_rt}
+        ./ci_util.py -b ${{matrix.build-type}}
+
+    - name: Create directory for SCM RT baselines
+      run: mkdir ${dir_bl}
+
+    - name: Download SCM RT baselines
+      run: |
+        cd ${dir_bl}
+        wget ftp://ftp.rap.ucar.edu:/pub/ccpp-scm/rt-baselines-${{matrix.build-type}}.zip
+        unzip rt-baselines-${{matrix.build-type}}.zip
+
+    - name: Compare SCM RT output to baselines
+      run: |
+        cd ${SCM_ROOT}/test
+        ./cmp_rt2bl.py --build_type ${{matrix.build-type}} --dir_rt ${dir_rt} --dir_bl ${dir_bl}
+
+    - name: Upload SCM RTs as GitHub Artifact
+      uses: actions/upload-artifact@v2
+      with: 
+        name: rt-baselines-${{matrix.build-type}}
+        path: ${dir_rt}

.github/workflows/ci_scm_ccpp_prebuild.yml

@@ -6,9 +6,7 @@ jobs:
   build-linux:
 
     # The type of runner that the job will run on
-    runs-on: ubuntu-latest
-    strategy:
-      max-parallel: 5
+    runs-on: ubuntu-20.04
 
     steps:
 
@@ -28,6 +26,9 @@ jobs:
         # $CONDA is an environment variable pointing to the root of the miniconda directory
         echo $CONDA/bin >> $GITHUB_PATH
 
+    - name: Update system packages
+      run: sudo apt-get update
+
     - name: Run ccpp_prebuild.py
       run: |
         mkdir -p /home/runner/work/ccpp-scm/ccpp-scm/scm/bin/ccpp/physics/physics/

test/ci_util.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python
+
+##############################################################################
+#
+# This script gathers the output from the $SCM_ROOT/ccpp-scm/scm/run directory
+# for the SCM CI Regression Tests. 
+#
+##############################################################################
+import os
+import sys
+from rt_test_cases import run_list
+from os.path import exists
+import argparse
+
+#
+parser = argparse.ArgumentParser()
+parser.add_argument('-b', '--build_type',  help='SCM build type')
+
+def parse_args():
+    args       = parser.parse_args()
+    build_type = args.build_type
+    return (build_type)
+
+def main():
+
+    (build_type) = parse_args()
+
+    #
+    for run in run_list:
+        case_tag = run["case"]+"_"+run["suite"]
+        file_out = "../scm/run/" + "output_"+case_tag+"/output.nc"
+        if exists(file_out):
+            os.system("mkdir -p artifact-"+build_type+"/"+case_tag+"/")
+            os.system("cp " + file_out + " artifact-"+build_type+"/"+case_tag+"/output.nc")
+        else:
+            print("FAIL: Could not copy output for baseline generation")
+            exit()
+
+if __name__ == '__main__':
+    main()

test/cmp_rt2bl.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python
+
+##############################################################################
+#
+# This script compares SCM RT output to baselines.
+#
+##############################################################################
+import os
+import sys
+from rt_test_cases import run_list
+from os.path import exists
+import argparse
+
+#
+parser = argparse.ArgumentParser()
+parser.add_argument('-b',   '--build_type',  help='SCM build type')
+parser.add_argument('-drt', '--dir_rt',      help='Directory containing SCM RT output')
+parser.add_argument('-dbl', '--dir_bl',      help='Directory containing SCM RT baselines')
+
+def parse_args():
+    args       = parser.parse_args()
+    build_type = args.build_type
+    dir_rt     = args.dir_rt 
+    dir_bl     = args.dir_bl
+
+    return (build_type,dir_rt,dir_bl)
+
+#
+def main():
+    #
+    (build_type, dir_rt, dir_bl) = parse_args()
+
+    #
+    error_count = 0
+    for run in run_list:
+        file_rt = dir_rt + "/" + run["case"]+"_"+run["suite"]+"/output.nc"
+        file_bl = dir_bl + "/" + run["case"]+"_"+run["suite"]+"/output.nc"
+        if exists(file_rt) and exists(file_bl):
+            com = "cmp "+file_rt+" "+file_bl+" > logfile.txt"
+            result = os.system(com)
+            if (result != 0):
+                print("Output for "+run["case"]+"_"+run["suite"]+ " DIFFERS from baseline")
+                error_count = error_count + 1
+            else:
+                print("Output for "+run["case"]+"_"+run["suite"]+ " is IDENTICAL to baseline")
+        else:
+            if not exists(file_rt):
+                print("Output for "+run["case"]+"_"+run["suite"]+ " is MISSING from output")
+            if not exists(file_bl):
+                print("Output for "+run["case"]+"_"+run["suite"]+ " is MISSING from baseline")
+            error_count = error_count + 1
+
+    #
+    if error_count == 0:
+        print("ALL TESTS PASSED, OUTPUT IS IDENTICAL.")
+    else:
+        print("ALL TESTS PASSED, BUT OUTPUT DIFFERS FROM BASELINE.")
+        #1/0
+
+#
+if __name__ == '__main__':
+    main()