Bring AutoRT updates into develop (#753)

Bring in AutoRT updates into develop from BrianCurtis-NOAA:feature/autort-bl

tests/auto/jobs/bl.py

@@ -11,6 +11,7 @@ def run(job_obj):
     pr_repo_loc, repo_dir_str = clone_pr_repo(job_obj, workdir)
     bldate = get_bl_date(job_obj, pr_repo_loc)
     bldir = f'{blstore}/develop-{bldate}/{job_obj.compiler.upper()}'
+    bldirbool = check_for_bl_dir(bldir, job_obj)
     run_regression_test(job_obj, pr_repo_loc)
     post_process(job_obj, pr_repo_loc, repo_dir_str, rtbldir, bldir)
 
@@ -24,8 +25,8 @@ def set_directories(job_obj):
                  f'REGRESSION_TEST_{job_obj.compiler.upper()}'
     elif job_obj.machine == 'jet':
         workdir = '/lfs4/HFIP/h-nems/emc.nemspara/autort/pr'
-        blstore = '/lfs4/HFIP/hfv3gfs/RT/NEMSfv3gfs/'
-        rtbldir = '/lfs4/HFIP/hfv3gfs/emc.nemspara/RT_BASELINE/'\
+        blstore = '/lfs4/HFIP/h-nems/emc.nemspara/RT/NEMSfv3gfs/'
+        rtbldir = '/lfs4/HFIP/h-nems/emc.nemspara/RT_BASELINE/'\
                  f'emc.nemspara/FV3_RT/REGRESSION_TEST_{job_obj.compiler.upper()}'
     elif job_obj.machine == 'gaea':
         workdir = '/lustre/f2/pdata/ncep/emc.nemspara/autort/pr'
@@ -38,9 +39,9 @@ def set_directories(job_obj):
         rtbldir = '/work/noaa/stmp/bcurtis/stmp/bcurtis/FV3_RT/'\
                  f'REGRESSION_TEST_{job_obj.compiler.upper()}'
     elif job_obj.machine == 'cheyenne':
-        workdir = '/glade/work/heinzell/fv3/ufs-weather-model/auto-rt'
-        blstore = '/glade/p/ral/jntp/GMTB/ufs-weather-model/RT'
-        rtbldir = '/glade/work/heinzell/FV3_RT/'\
+        workdir = '/glade/scratch/dtcufsrt/autort/tests/auto/pr'
+        blstore = '/glade/p/ral/jntp/GMTB/ufs-weather-model/RT/NEMSfv3gfs'
+        rtbldir = '/glade/scratch/dtcufsrt/FV3_RT/'\
                  f'REGRESSION_TEST_{job_obj.compiler.upper()}'
     else:
         logger.critical(f'Machine {job_obj.machine} is not supported for this job')
@@ -54,18 +55,20 @@ def set_directories(job_obj):
     return workdir, rtbldir, blstore
 
 
-def check_for_bl_dir(bldir):
+def check_for_bl_dir(bldir, job_obj):
     logger = logging.getLogger('BL/CHECK_FOR_BL_DIR')
     logger.info('Checking if baseline directory exists')
     if os.path.exists(bldir):
         logger.critical(f'Baseline dir: {bldir} exists. It should not, yet.')
+        job_obj.comment_text_append(f'{bldir}\n Exists already. '
+                                    'It should not yet. Please delete.')
         raise FileExistsError
     return False
 
 
-def create_bl_dir(bldir):
+def create_bl_dir(bldir, job_obj):
     logger = logging.getLogger('BL/CREATE_BL_DIR')
-    if not check_for_bl_dir(bldir):
+    if not check_for_bl_dir(bldir, job_obj):
         os.makedirs(bldir)
         if not os.path.exists(bldir):
             logger.critical(f'Someting went wrong creating {bldir}')
@@ -154,9 +157,12 @@ def post_process(job_obj, pr_repo_loc, repo_dir_str, rtbldir, bldir):
     filepath = f'{pr_repo_loc}/{rt_log}'
     rt_dir, logfile_pass = process_logfile(job_obj, filepath)
     if logfile_pass:
-        create_bl_dir(bldir)
+        create_bl_dir(bldir, job_obj)
         move_bl_command = [[f'mv {rtbldir}/* {bldir}/', pr_repo_loc]]
+        if job_obj.machine == 'orion':
+            move_bl_command.append([f'/bin/bash --login adjust_permissions.sh orion develop-{bldate}', blstore])
         job_obj.run_commands(logger, move_bl_command)
+        job_obj.comment_text_append('Baseline creation and move successful')
         logger.info('Starting RT Job')
         rt.run(job_obj)
         logger.info('Finished with RT Job')
@@ -196,10 +202,12 @@ def get_bl_date(job_obj, pr_repo_loc):
 def process_logfile(job_obj, logfile):
     logger = logging.getLogger('BL/PROCESS_LOGFILE')
     rt_dir = []
+    fail_string_list = ['Test', 'failed']
     if os.path.exists(logfile):
         with open(logfile) as f:
             for line in f:
-                if 'FAIL' in line and 'Test' in line:
+                if all(x in line for x in fail_string_list):
+                # if 'FAIL' in line and 'Test' in line:
                     job_obj.comment_text_append(f'{line.rstrip(chr(10))}')
                 elif 'working dir' in line and not rt_dir:
                     logger.info(f'Found "working dir" in line: {line}')

tests/auto/jobs/rt.py

@@ -23,7 +23,7 @@ def set_directories(job_obj):
     elif job_obj.machine == 'orion':
         workdir = '/work/noaa/nems/emc.nemspara/autort/pr'
     elif job_obj.machine == 'cheyenne':
-        workdir = '/glade/work/heinzell/fv3/ufs-weather-model/auto-rt'
+        workdir = '/glade/scratch/dtcufsrt/autort/tests/auto/pr'
     else:
         print(f'Machine {job_obj.machine} is not supported for this job')
         raise KeyError
@@ -94,26 +94,36 @@ def post_process(job_obj, pr_repo_loc, repo_dir_str, branch):
     filepath = f'{pr_repo_loc}/{rt_log}'
     rt_dir, logfile_pass = process_logfile(job_obj, filepath)
     if logfile_pass:
-        move_rt_commands = [
-            [f'git pull --ff-only origin {branch}', pr_repo_loc],
-            [f'git add {rt_log}', pr_repo_loc],
-            [f'git commit -m "RT JOBS PASSED: {job_obj.machine}'
-             f'.{job_obj.compiler}. Log file uploaded."',
-             pr_repo_loc],
-            ['sleep 10', pr_repo_loc],
-            [f'git push origin {branch}', pr_repo_loc]
-        ]
-        job_obj.run_commands(logger, move_rt_commands)
-        remove_pr_data(job_obj, pr_repo_loc, repo_dir_str, rt_dir)
+        if job_obj.preq_dict['preq'].maintainer_can_modify:
+            move_rt_commands = [
+                [f'git pull --ff-only origin {branch}', pr_repo_loc],
+                [f'git add {rt_log}', pr_repo_loc],
+                [f'git commit -m "RT JOBS PASSED: {job_obj.machine}'
+                 f'.{job_obj.compiler}. Log file uploaded.\n\n'
+                  'on-behalf-of @ufs-community"',
+                 pr_repo_loc],
+                ['sleep 10', pr_repo_loc],
+                [f'git push origin {branch}', pr_repo_loc]
+            ]
+            job_obj.run_commands(logger, move_rt_commands)
+            remove_pr_data(job_obj, pr_repo_loc, repo_dir_str, rt_dir)
+        else:
+            job_obj.comment_text_append(f'Cannot upload {job_obj.machine}.'\
+                                        f'{job_obj.compiler} RT Log'\
+                                        'It is blocked by PR owner')
+            job_obj.comment_text_append(f'Please obtain logs from {pr_repo_loc}')
+            job_obj.preq_dict['preq'].create_issue_comment(job_obj.comment_text)
 
 
 def process_logfile(job_obj, logfile):
     logger = logging.getLogger('RT/PROCESS_LOGFILE')
     rt_dir = []
+    fail_string_list = ['Test', 'failed']
     if os.path.exists(logfile):
         with open(logfile) as f:
             for line in f:
-                if 'FAIL' in line and 'Test' in line:
+                if all(x in line for x in fail_string_list):
+                # if 'FAIL' in line and 'Test' in line:
                     job_obj.comment_text_append(f'{line.rstrip(chr(10))}')
                 elif 'working dir' in line and not rt_dir:
                     rt_dir = os.path.split(line.split()[-1])[0]

tests/auto/rt_auto.py

@@ -212,6 +212,8 @@ def setup_env():
     hostname = os.getenv('HOSTNAME')
     if bool(re.match(re.compile('hfe.+'), hostname)):
         machine = 'hera'
+    elif bool(re.match(re.compile('hecflow.+'), hostname)):
+        machine = 'hera'
     elif bool(re.match(re.compile('fe.+'), hostname)):
         machine = 'jet'
         os.environ['ACCNR'] = 'h-nems'
@@ -220,7 +222,7 @@ def setup_env():
         os.environ['ACCNR'] = 'nggps_emc'
     elif bool(re.match(re.compile('Orion-login.+'), hostname)):
         machine = 'orion'
-    elif bool(re.match(re.compile('.+.cheyenne.ucar.edu'), hostname)):
+    elif bool(re.match(re.compile('chadmin.+'), hostname)):
         machine = 'cheyenne'
         os.environ['ACCNR'] = 'P48503002'
     else:

tests/auto/start_rt_auto.sh

@@ -4,6 +4,9 @@ set -eux
 if [[ $HOSTNAME == hfe* ]]; then
   export PATH=/scratch1/NCEPDEV/nems/emc.nemspara/soft/miniconda3/bin:$PATH
   export PYTHONPATH=/scratch1/NCEPDEV/nems/emc.nemspara/soft/miniconda3/lib/python3.8/site-packages
+elif [[ $HOSTNAME == hecflow* ]]; then
+  export PATH=/scratch1/NCEPDEV/nems/emc.nemspara/soft/miniconda3/bin:$PATH
+  export PYTHONPATH=/scratch1/NCEPDEV/nems/emc.nemspara/soft/miniconda3/lib/python3.8/site-packages
 elif [[ $HOSTNAME == Orion-login-* ]]; then
   export PATH=/work/noaa/nems/emc.nemspara/soft/miniconda3/bin:$PATH
   export PYTHONPATH=/work/noaa/nems/emc.nemspara/soft/miniconda3/lib/python3.8/site-packages
@@ -13,7 +16,8 @@ elif [[ $HOSTNAME == fe* ]]; then
 elif [[ $HOSTNAME == gaea* ]]; then
   export PATH=/lustre/f2/pdata/esrl/gsd/contrib/miniconda3/4.8.3/envs/ufs-weather-model/bin:$PATH
   export PYTHONPATH=/lustre/f2/pdata/esrl/gsd/contrib/miniconda3/4.8.3/lib/python3.8/site-packages
-elif [[ $HOSTNAME == *cheyenne* ]]; then
+elif [[ $HOSTNAME == *chadmin* ]]; then
+  export MACHINE_ID=cheyenne
   export PATH=/glade/p/ral/jntp/tools/miniconda3/4.8.3/envs/ufs-weather-model/bin:/glade/p/ral/jntp/tools/miniconda3/4.8.3/bin:$PATH
   export PYTHONPATH=/glade/p/ral/jntp/tools/miniconda3/4.8.3/envs/ufs-weather-model/lib/python3.8/site-packages:/glade/p/ral/jntp/tools/miniconda3/4.8.3/lib/python3.8/site-packages
 else