Update CI driver (#1401)

# Description

This PR consolidates the ```run_ci.sh``` and ```run_gw_ci.sh``` as well
as the ```driver.sh``` and ```gw_driver.sh``` scripts. Now, a ```-w```
option indicates whether testing is taking place in the Global Workflow.

A ```-E ci_test1|ci_test2|...``` is added to ```driver.sh``` to exclude
CI tests which don't need to be run. Those options are passed directly
to the ```ctest``` command. The PR testing functionality in
```driver.sh``` knows which tests to exclude based on the check boxes
below. The output from the EMC bot for PR testing will now show the
exact CTest command to indicate which tests were indeed excluded, just
in case the check boxes are changed after the CI testing has begun.

A PR template is now added for GDASApp that includes the CI testing
checkboxes.

You can see below that I've tested the PR testing functionality.

# Companion PRs

<!-- Enter links to any companion PRs here. -->

# Issues

<!-- Enter any issues referenced or resolved by this PR here. Use
keywords "Resolves" or "Refs".
Resolves #1234
Refs #4321
Refs NOAA-EMC/repo#5678
-->

# Automated CI tests to run in Global Workflow
<!-- Which Global Workflow CI tests are required to adequately test this
PR? -->
- [x] atm_jjob <!-- JEDI atm single cycle DA !-->
- [x] C96C48_ufs_hybatmDA <!-- JEDI atm cycled DA !-->
- [x] C96C48_hybatmaerosnowDA  <!-- JEDI aero/snow cycled DA !-->
- [x] C48mx500_3DVarAOWCDA <!-- JEDI low-res marine 3DVar cycled DA !-->
- [x] C48mx500_hybAOWCDA <!-- JEDI marine hybrid envar cycled DA !-->
- [ ] C96C48_hybatmDA <!-- GSI atm cycled DA !-->

---------

Co-authored-by: RussTreadon-NOAA <[email protected]>

.github/pull_request_template.md

@@ -0,0 +1,24 @@
+# Description
+
+<!-- Enter PR description here. -->
+
+# Companion PRs
+
+<!-- Enter links to any companion PRs here. -->
+
+# Issues
+
+<!-- Enter any issues referenced or resolved by this PR here. Use keywords "Resolves" or "Refs".
+Resolves #1234
+Refs #4321
+Refs NOAA-EMC/repo#5678
+-->
+
+# Automated CI tests to run in Global Workflow
+<!-- Which Global Workflow CI tests are required to adequately test this PR? -->
+- [ ] atm_jjob <!-- JEDI atm single cycle DA !-->
+- [ ] C96C48_ufs_hybatmDA <!-- JEDI atm cycled DA !-->
+- [ ] C96C48_hybatmaerosnowDA  <!-- JEDI aero/snow cycled DA !-->
+- [ ] C48mx500_3DVarAOWCDA <!-- JEDI low-res marine 3DVar cycled DA  !-->
+- [ ] C48mx500_hybAOWCDA <!-- JEDI marine hybrid envar cycled DA !-->
+- [ ] C96C48_hybatmDA <!-- GSI atm cycled DA !-->

ci/ci_tests.sh

@@ -0,0 +1,6 @@
+CI_TESTS=("atm_jjob"
+          "C96C48_ufs_hybatmDA"
+          "C96C48_hybatmaerosnowDA"
+          "C48mx500_3DVarAOWCDA"
+          "C48mx500_hybAOWCDA"
+          "C96C48_hybatmDA")

ci/driver.sh

@@ -1,6 +1,6 @@
 #!/bin/bash --login
 
-echo "Start at $(date)"
+echo "Starting automated testing at $(date)"
 
 my_dir="$( cd "$( dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd )"
 echo "Set my_dir ${my_dir}"
@@ -13,6 +13,7 @@ usage() {
   echo
   echo "  -t  target/machine script is running on    DEFAULT: $(hostname)"
   echo "  -h  display this message and quit"
+  echo "  -w  run workflow tests on $(hostname)"
   echo
   exit 1
 }
@@ -22,20 +23,25 @@ usage() {
 
 export TARGET="$(hostname)"
 
-while getopts "t:h" opt; do
+TEST_WORKFLOW=0
+while getopts "t:h:w" opt; do
   case $opt in
     t)
       TARGET=$OPTARG
       ;;
     h|\?|:)
       usage
       ;;
+    w)
+      TEST_WORKFLOW=1
+      ;;  
   esac
 done
 
+echo "Running automated testing on $TARGET"
+
 case ${TARGET} in
-  hera | orion)
-    echo "Running Automated Testing on $TARGET"
+  hera | orion | hercules)
     source $MODULESHOME/init/sh
     source $my_dir/${TARGET}.sh
     module purge
@@ -49,106 +55,176 @@ case ${TARGET} in
     ;;
 esac
 
+# ==============================================================================
+# set list of available CI tests to run on the Global Workflow
+source $my_dir/ci_tests.sh
+
+# ==============================================================================
+# set things that depend on whether running workflow tests or not
+gdasapp_url="https://github.com/NOAA-EMC/GDASApp.git"
+if [[ $TEST_WORKFLOW == 1 ]]; then
+  echo "Testing GDASApp inside the Global Workflow"
+
+  CI_LABEL="${GDAS_CI_HOST}-GW-RT"
+  OPEN_PR_LIST_DIR=$GDAS_CI_ROOT/open_pr_list_gw
+  PR_TEST_DIR=$GDAS_CI_ROOT/workflow/PR
+  BASE_REPO=global-workflow
+
+  # Default Global Workflow repo and branch if no companion PR found
+  workflow_url="https://github.com/NOAA-EMC/global-workflow.git"
+  workflow_branch="develop"
+else
+  echo "Testing stand-alone GDASApp"
+
+  CI_LABEL="${GDAS_CI_HOST}-RT"
+  OPEN_PR_LIST_DIR=$GDAS_CI_ROOT/open_pr_list
+  PR_TEST_DIR=$GDAS_CI_ROOT/PR
+  BASE_REPO=GDASApp
+fi
+
 # ==============================================================================
 # pull on the repo and get list of open PRs
+
 cd $GDAS_CI_ROOT/repo
-CI_LABEL="${GDAS_CI_HOST}-RT"
-gh pr list --label "$CI_LABEL" --state "open" | awk '{print $1;}' > $GDAS_CI_ROOT/open_pr_list
 
-open_pr=`cat $GDAS_CI_ROOT/open_pr_list | wc -l`
+gh pr list --label "$CI_LABEL" --state "open" | awk '{print $1;}' > $OPEN_PR_LIST_DIR
+
+open_pr=`cat $OPEN_PR_LIST_DIR | wc -l`
 if (( $open_pr == 0 )); then
   echo "No open PRs with ${CI_LABEL}, exit."
-  echo "Finish at $(date)"    
+  echo "Finished automated testing at $(date)"    
   exit
 fi
 
-open_pr_list=$(cat $GDAS_CI_ROOT/open_pr_list)
+open_pr_list=$(cat $OPEN_PR_LIST_DIR)
 
 # ==============================================================================
 # clone, checkout, build, test, etc.
-repo_url="https://github.com/NOAA-EMC/GDASApp.git"
 # loop through all open PRs
 for pr in $open_pr_list; do
   echo " "
-  echo "Start processing Pull Request #${pr} at $(date)"
+  echo "Starting processing of pull request #${pr} at $(date)"
 
   # get the branch name used for the PR
   gdasapp_branch=$(gh pr view $pr --json headRefName -q ".headRefName")
 
-  # get additional branch information
-  branch_owner=$(gh pr view $pr --repo ${repo_url} --json headRepositoryOwner --jq '.headRepositoryOwner.login')
-  branch_name=$(gh pr view $pr --repo ${repo_url} --json headRepository --jq '.headRepository.name')
-  pr_assignees=$(gh pr view $pr --repo ${repo_url} --json assignees --jq '.assignees[].login')
-
+  # get additional branch informatio
+  branch_owner=$(gh pr view $pr --repo ${gdasapp_url} --json headRepositoryOwner --jq '.headRepositoryOwner.login')
+  branch_name=$(gh pr view $pr --repo ${gdasapp_url} --json headRepository --jq '.headRepository.name')
+  pr_assignees=$(gh pr view $pr --repo ${gdasapp_url} --json assignees --jq '.assignees[].login')
+  
   # check if any assignee is authorized to run CI
-  authorized_by=""
+  rc=1
   for str in ${pr_assignees[@]}; do
-    grep $str /scratch1/NCEPDEV/da/role.jedipara/CI/GDASApp/authorized_users
-    rc=$?
+    grep $str $AUTHORIZED_USERS_FILE > /dev/null
+    if (( rc != 0 )); then
+	rc=$?
+    fi	
     if (( rc == 0 )); then
-      authorized_by=${str}
-      echo "FOUND MATCH $str, rc $rc"
-      break
+      echo "Authorized user $str assigned to this PR"
     fi
   done
 
   # Authorized to run CI
   if (( rc == 0 )); then
-    echo "Run CI"
+    echo "CI authorized. Running CI..."
 
     # update PR label
     gh pr edit $pr --remove-label $CI_LABEL --add-label ${CI_LABEL}-Running
-
-    # construct the fork URL
-    gdasapp_url="https://github.com/$branch_owner/${branch_name}.git"
-
+
     echo "GDASApp URL: $gdasapp_url"
     echo "GDASApp branch Name: $gdasapp_branch"
-    echo "CI authorized by $authorized_by at $(date)"
+
+    if [[ $TEST_WORKFLOW == 1 ]]; then
+      # check for a companion PR in the global-workflow
+      companion_pr_exists=$(gh pr list --repo ${workflow_url} --head ${gdasapp_branch} --state open)
+
+      if [ -n "$companion_pr_exists" ]; then
+        # get the PR number
+        companion_pr=$(echo "$companion_pr_exists" | awk '{print $1;}')
+
+        # extract the necessary info
+        branch_owner=$(gh pr view $companion_pr --repo $workflow_url --json headRepositoryOwner --jq '.headRepositoryOwner.login')
+        branch_name=$(gh pr view $companion_pr --repo $workflow_url --json headRepository --jq '.headRepository.name')
+
+        # Construct fork URL. Update workflow branch name
+        workflow_url="https://github.com/$branch_owner/$branch_name.git"
+        workflow_branch=$gdasapp_branch
+      fi
+
+      echo "Found companion Global Workflow PR #${companion_pr}!"
+      echo "Global Workflow URL: $workflow_url"
+      echo "Global Workflow branch name: $workflow_branch"      
+    fi
 
-    # create PR specific directory
-    if [ -d $GDAS_CI_ROOT/PR/$pr ]; then
-        rm -rf $GDAS_CI_ROOT/PR/$pr
+    # create PR specific directory    
+    if [ -d $PR_TEST_DIR/$pr ]; then
+        rm -rf $PR_TEST_DIR/$pr
     fi
-    mkdir -p $GDAS_CI_ROOT/PR/$pr
-    cd $GDAS_CI_ROOT/PR/$pr
+    mkdir -p $PR_TEST_DIR/$pr
+    cd $PR_TEST_DIR/$pr        
     pwd
 
     # clone copy of repo
-    git clone --recursive --jobs 8 --branch $gdasapp_branch $gdasapp_url
-    cd GDASApp
+    if [[ $TEST_WORKFLOW == 1 ]]; then
+      echo "Cloning Global Workflow branch $workflow_branch from $workflow_url at $(date)"
+      git clone --recursive --jobs 8 --branch $workflow_branch $workflow_url
+      cd global-workflow/sorc/gdas.cd
+    else
+      echo "Cloning GDASApp branch $workflow_branch at $(date)"	
+      git clone --recursive --jobs 8 --branch $gdasapp_branch $gdasapp_url
+      cd GDASApp
+    fi
     pwd
 
     # checkout GDASApp pull request
-    git pull
     gh pr checkout $pr
     git submodule update --init --recursive
 
     # get commit hash
     commit=$(git log --pretty=format:'%h' -n 1)
-    echo "$commit" > $GDAS_CI_ROOT/PR/$pr/commit
+    echo "$commit" > $PR_TEST_DIR/$pr/commit
 
     # run build and testing command
-    echo "Execute $my_dir/run_ci.sh for $GDAS_CI_ROOT/PR/$pr/GDASApp at $(date)"
-    $my_dir/run_ci.sh -d $GDAS_CI_ROOT/PR/$pr/GDASApp -o $GDAS_CI_ROOT/PR/$pr/output_${commit}
+    echo "Running run_ci.sh for $PR_TEST_DIR/$pr/$BASE_REPO at $(date)"
+    run_ci_cmd="$my_dir/run_ci.sh -d $PR_TEST_DIR/$pr/$BASE_REPO -o $PR_TEST_DIR/$pr/output_${commit}"
+    if [[ $TEST_WORKFLOW == 1 ]]; then
+      # get ci tests from PR description and convert into a regular expressions to be excluded
+      branch_body=$(gh pr view $pr --repo ${gdasapp_url} --json body --jq '.body')
+      ci_checklist=$(echo "$branch_body" | grep '\[x\]')
+      ctest_regex_exclude=""  
+      for ci_test in ${CI_TESTS[@]}; do
+        if ! echo "$ci_checklist" | grep -q "$ci_test"; then
+	  ctest_regex_exclude+="${ctest_regex_exclude:+|}$ci_test"
+	fi      
+      done
+
+      # setup run_ci.sh arguments to test in the Global Workflow and exclude chosen CI tests
+      run_ci_cmd+=" -w"
+      if [ -n "$ctest_regex_exclude" ]; then
+        run_ci_cmd+=" -E $ctest_regex_exclude"
+      fi
+    fi
+    $run_ci_cmd
     ci_status=$?
-    echo "After run_ci.sh with ci_status ${ci_status} at $(date)"
-    gh pr comment $pr --repo ${repo_url} --body-file $GDAS_CI_ROOT/PR/$pr/output_${commit}
+    echo "Finished running run_ci.sh with ci_status ${ci_status} at $(date)"
+
+    gh pr comment $pr --repo ${gdasapp_url} --body-file $PR_TEST_DIR/$pr/output_${commit}
     if [ $ci_status -eq 0 ]; then
-      gh pr edit $pr --repo ${repo_url} --remove-label ${CI_LABEL}-Running --add-label ${CI_LABEL}-Passed
+      gh pr edit $pr --repo ${gdasapp_url} --remove-label ${CI_LABEL}-Running --add-label ${CI_LABEL}-Passed
     else
-      gh pr edit $pr --repo ${repo_url} --remove-label ${CI_LABEL}-Running --add-label ${CI_LABEL}-Failed
+      gh pr edit $pr --repo ${gdasapp_url} --remove-label ${CI_LABEL}-Running --add-label ${CI_LABEL}-Failed
     fi
 
   # Not authorized to run CI
   else
-    echo "Do NOT run CI"
+    echo "No authorized users assigned to this PR. Aborting CI..."
   fi
 
-  echo "Finish processing Pull Request #{pr} at $(date)"
+  echo "Finished processing Pull Request #${pr} at $(date)"
 done
 
 # ==============================================================================
 # scrub working directory for older files
-find $GDAS_CI_ROOT/PR/* -maxdepth 1 -mtime +3 -exec rm -rf {} \;
-echo "Finish at $(date)"
+find $PR_TEST_DIR/* -maxdepth 1 -mtime +3 -exec rm -rf {} \;
+echo "Finished automated testing at $(date)"