From 12696d75c4bf98c07f531e56f4bf6a3f0955a705 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Fri, 17 Jan 2025 19:24:08 -0500 Subject: [PATCH 001/105] initial modification to split arch job to two parts --- .../{JGLOBAL_ARCHIVE => JGLOBAL_ARCHIVE_TARS} | 6 +- jobs/JGLOBAL_ARCHIVE_VRFY | 79 +++++++++++++++++++ jobs/rocoto/{arch.sh => arch_tars.sh} | 4 +- jobs/rocoto/arch_vrfy.sh | 24 ++++++ parm/config/gefs/config.base | 3 + parm/config/gefs/yaml/defaults.yaml | 1 + ...al_archive.py => exglobal_archive_tars.py} | 0 scripts/exglobal_archive_vrfy.py | 74 +++++++++++++++++ workflow/applications/applications.py | 1 + workflow/applications/gefs.py | 5 +- workflow/rocoto/gefs_tasks.py | 50 +++++++++++- workflow/rocoto/tasks.py | 4 +- 12 files changed, 240 insertions(+), 11 deletions(-) rename jobs/{JGLOBAL_ARCHIVE => JGLOBAL_ARCHIVE_TARS} (93%) mode change 100755 => 100644 create mode 100644 jobs/JGLOBAL_ARCHIVE_VRFY rename jobs/rocoto/{arch.sh => arch_tars.sh} (90%) mode change 100755 => 100644 create mode 100644 jobs/rocoto/arch_vrfy.sh rename scripts/{exglobal_archive.py => exglobal_archive_tars.py} (100%) mode change 100755 => 100644 create mode 100644 scripts/exglobal_archive_vrfy.py diff --git a/jobs/JGLOBAL_ARCHIVE b/jobs/JGLOBAL_ARCHIVE_TARS old mode 100755 new mode 100644 similarity index 93% rename from jobs/JGLOBAL_ARCHIVE rename to jobs/JGLOBAL_ARCHIVE_TARS index 401feba35f..96a9b863bc --- a/jobs/JGLOBAL_ARCHIVE +++ b/jobs/JGLOBAL_ARCHIVE_TARS @@ -1,7 +1,7 @@ #! /usr/bin/env bash source "${HOMEgfs}/ush/preamble.sh" -source "${HOMEgfs}/ush/jjob_header.sh" -e "arch" -c "base arch" +source "${HOMEgfs}/ush/jjob_header.sh" -e "arch_tars" -c "base arch_tars" ############################################## @@ -33,8 +33,8 @@ YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ COMIN_OCEAN_GRIB:COM_OCEAN_GRIB_TMPL \ COMIN_OCEAN_NETCDF:COM_OCEAN_NETCDF_TMPL \ COMIN_OCEAN_ANALYSIS:COM_OCEAN_ANALYSIS_TMPL \ - COMIN_OCEAN_BMATRIX:COM_OCEAN_BMATRIX_TMPL \ - COMIN_ICE_BMATRIX:COM_ICE_BMATRIX_TMPL \ + COMIN_OCEAN_BMATRIX:COM_OCEAN_BMATRIX_TMPL \ + COMIN_ICE_BMATRIX:COM_ICE_BMATRIX_TMPL \ COMIN_WAVE_GRID:COM_WAVE_GRID_TMPL \ COMIN_WAVE_HISTORY:COM_WAVE_HISTORY_TMPL \ COMIN_WAVE_STATION:COM_WAVE_STATION_TMPL \ diff --git a/jobs/JGLOBAL_ARCHIVE_VRFY b/jobs/JGLOBAL_ARCHIVE_VRFY new file mode 100644 index 0000000000..7fab09f646 --- /dev/null +++ b/jobs/JGLOBAL_ARCHIVE_VRFY @@ -0,0 +1,79 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" +source "${HOMEgfs}/ush/jjob_header.sh" -e "arch_vrfy" -c "base arch_vrfy" + + +############################################## +# Set variables used in the script +############################################## +YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ + COMIN_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL \ + COMIN_ATMOS_BUFR:COM_ATMOS_BUFR_TMPL \ + COMIN_ATMOS_GEMPAK:COM_ATMOS_GEMPAK_TMPL \ + COMIN_ATMOS_GENESIS:COM_ATMOS_GENESIS_TMPL \ + COMIN_ATMOS_HISTORY:COM_ATMOS_HISTORY_TMPL \ + COMIN_ATMOS_INPUT:COM_ATMOS_INPUT_TMPL \ + COMIN_ATMOS_MASTER:COM_ATMOS_MASTER_TMPL \ + COMIN_ATMOS_RESTART:COM_ATMOS_RESTART_TMPL \ + COMIN_ATMOS_TRACK:COM_ATMOS_TRACK_TMPL \ + COMIN_ATMOS_WMO:COM_ATMOS_WMO_TMPL \ + COMIN_CHEM_HISTORY:COM_CHEM_HISTORY_TMPL \ + COMIN_CHEM_ANALYSIS:COM_CHEM_ANALYSIS_TMPL \ + COMIN_MED_RESTART:COM_MED_RESTART_TMPL \ + COMIN_SNOW_ANALYSIS:COM_SNOW_ANALYSIS_TMPL \ + COMIN_ICE_HISTORY:COM_ICE_HISTORY_TMPL \ + COMIN_ICE_INPUT:COM_ICE_INPUT_TMPL \ + COMIN_ICE_RESTART:COM_ICE_RESTART_TMPL \ + COMIN_ICE_GRIB:COM_ICE_GRIB_TMPL \ + COMIN_OBS:COM_OBS_TMPL \ + COMIN_TOP:COM_TOP_TMPL \ + COMIN_OCEAN_HISTORY:COM_OCEAN_HISTORY_TMPL \ + COMIN_OCEAN_RESTART:COM_OCEAN_RESTART_TMPL \ + COMIN_OCEAN_GRIB:COM_OCEAN_GRIB_TMPL \ + COMIN_OCEAN_NETCDF:COM_OCEAN_NETCDF_TMPL \ + COMIN_OCEAN_ANALYSIS:COM_OCEAN_ANALYSIS_TMPL \ + COMIN_OCEAN_BMATRIX:COM_OCEAN_BMATRIX_TMPL \ + COMIN_ICE_BMATRIX:COM_ICE_BMATRIX_TMPL \ + COMIN_WAVE_GRID:COM_WAVE_GRID_TMPL \ + COMIN_WAVE_HISTORY:COM_WAVE_HISTORY_TMPL \ + COMIN_WAVE_STATION:COM_WAVE_STATION_TMPL \ + COMIN_WAVE_RESTART:COM_WAVE_RESTART_TMPL \ + COMIN_ATMOS_OZNMON:COM_ATMOS_OZNMON_TMPL \ + COMIN_ATMOS_RADMON:COM_ATMOS_RADMON_TMPL \ + COMIN_ATMOS_MINMON:COM_ATMOS_MINMON_TMPL \ + COMIN_CONF:COM_CONF_TMPL \ + COMOUT_ATMOS_TRACK:COM_ATMOS_TRACK_TMPL + +for grid in "0p25" "0p50" "1p00"; do + YMD=${PDY} HH=${cyc} GRID=${grid} declare_from_tmpl -rx \ + "COMIN_ATMOS_GRIB_${grid}:COM_ATMOS_GRIB_GRID_TMPL" +done + +############################################################### +# Run archive script +############################################################### + +${GLOBALARCHIVESH:-${SCRgfs}/exglobal_archive.py} +status=$? +[[ ${status} -ne 0 ]] && exit "${status}" + +############################################## +# End JOB SPECIFIC work +############################################## + +############################################## +# Final processing +############################################## +if [[ -e "${pgmout}" ]] ; then + cat "${pgmout}" +fi + + +########################################## +# Remove the Temporary working directory +########################################## +cd "${DATAROOT}" || (echo "${DATAROOT} does not exist. ABORT!"; exit 1) +[[ ${KEEPDATA} = "NO" ]] && rm -rf "${DATA}" + +exit 0 diff --git a/jobs/rocoto/arch.sh b/jobs/rocoto/arch_tars.sh old mode 100755 new mode 100644 similarity index 90% rename from jobs/rocoto/arch.sh rename to jobs/rocoto/arch_tars.sh index 083e319bf5..d9c58a9b38 --- a/jobs/rocoto/arch.sh +++ b/jobs/rocoto/arch_tars.sh @@ -13,12 +13,12 @@ status=$? PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${HOMEgfs}/ush/python" export PYTHONPATH -export job="arch" +export job="arch_tars" export jobid="${job}.$$" ############################################################### # Execute the JJOB -"${HOMEgfs}"/jobs/JGLOBAL_ARCHIVE +"${HOMEgfs}"/jobs/JGLOBAL_ARCHIVE_TARS status=$? exit "${status}" diff --git a/jobs/rocoto/arch_vrfy.sh b/jobs/rocoto/arch_vrfy.sh new file mode 100644 index 0000000000..fee66c01b6 --- /dev/null +++ b/jobs/rocoto/arch_vrfy.sh @@ -0,0 +1,24 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" + +############################################################### +# Source FV3GFS workflow modules +. "${HOMEgfs}"/ush/load_fv3gfs_modules.sh +status=$? +[[ ${status} -ne 0 ]] && exit "${status}" + +############################################################### +# setup python path for workflow utilities and tasks +PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${HOMEgfs}/ush/python" +export PYTHONPATH + +export job="arch_vrfy" +export jobid="${job}.$$" + +############################################################### +# Execute the JJOB +"${HOMEgfs}"/jobs/JGLOBAL_ARCHIVE_VRFY +status=$? + +exit "${status}" diff --git a/parm/config/gefs/config.base b/parm/config/gefs/config.base index fc641fef6b..884afbf89e 100644 --- a/parm/config/gefs/config.base +++ b/parm/config/gefs/config.base @@ -333,11 +333,14 @@ export DO_METP="NO" # Run METPLUS jobs - set METPLUS settings in config. export DO_FIT2OBS="NO" # Run fit to observations package # Archiving options +export VRFYARCH="@VRFYARCH@" # save verification data locally export HPSSARCH="@HPSSARCH@" # save data to HPSS archive export LOCALARCH="@LOCALARCH@" # save data to local archive if [[ ${HPSSARCH} = "YES" ]] && [[ ${LOCALARCH} = "YES" ]]; then echo "Both HPSS and local archiving selected. Please choose one or the other." exit 3 +elif [[ ${HPSSARCH} = "YES" ]] || [[ ${LOCALARCH} = "YES" ]]; then + export DO_ARCHTAR="YES" fi export ARCH_CYC=00 # Archive data at this cycle for warm start and/or forecast-only capabilities export ARCH_WARMICFREQ=4 # Archive frequency in days for warm start capability diff --git a/parm/config/gefs/yaml/defaults.yaml b/parm/config/gefs/yaml/defaults.yaml index df12b16282..9347ecad3e 100644 --- a/parm/config/gefs/yaml/defaults.yaml +++ b/parm/config/gefs/yaml/defaults.yaml @@ -19,6 +19,7 @@ base: FHOUT_ICE_GFS: 6 HPSSARCH: "NO" LOCALARCH: "NO" + VRFYARCH: "YES" SFS_POST: "NO" USE_OCN_ENS_PERTURB_FILES: "NO" USE_ATM_ENS_PERTURB_FILES: "NO" diff --git a/scripts/exglobal_archive.py b/scripts/exglobal_archive_tars.py old mode 100755 new mode 100644 similarity index 100% rename from scripts/exglobal_archive.py rename to scripts/exglobal_archive_tars.py diff --git a/scripts/exglobal_archive_vrfy.py b/scripts/exglobal_archive_vrfy.py new file mode 100644 index 0000000000..df49704e06 --- /dev/null +++ b/scripts/exglobal_archive_vrfy.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 + +import os + +from pygfs.task.archive import Archive +from wxflow import AttrDict, Logger, cast_strdict_as_dtypedict, logit, chdir + +# initialize root logger +logger = Logger(level=os.environ.get("LOGGING_LEVEL", "DEBUG"), colored_log=True) + + +@logit(logger) +def main(): + + config = cast_strdict_as_dtypedict(os.environ) + + # Instantiate the Archive object + archive = Archive(config) + + # update these keys to be 3 digits if they are part of archive.task_config.keys + for key in ['OCNRES', 'ICERES']: + try: + archive.task_config[key] = f"{archive.task_config[key]:03d}" + except KeyError as ee: + logger.info(f"key ({key}) not found in archive.task_config!") + + # Pull out all the configuration keys needed to run the rest of archive steps + keys = ['ATARDIR', 'current_cycle', 'FHMIN', 'FHMAX', 'FHOUT', 'RUN', 'PDY', + 'DO_VERFRAD', 'DO_VMINMON', 'DO_VERFOZN', 'DO_ICE', 'DO_PREP_OBS_AERO', + 'PARMgfs', 'DO_OCN', 'DO_WAVE', 'WRITE_DOPOST', 'PSLOT', 'HPSSARCH', 'DO_MOS', + 'DO_JEDISNOWDA', 'LOCALARCH', 'REALTIME', 'ROTDIR', 'ARCH_WARMICFREQ', + 'ARCH_FCSTICFREQ', 'ARCH_CYC', 'assim_freq', 'ARCDIR', 'SDATE', + 'FHMIN_GFS', 'FHMAX_GFS', 'FHOUT_GFS', 'ARCH_GAUSSIAN', 'MODE', + 'FHOUT_OCN', 'FHOUT_ICE', 'FHOUT_OCN_GFS', 'FHOUT_ICE_GFS', 'DO_BUFRSND', 'DOHYBVAR', + 'ARCH_GAUSSIAN_FHMAX', 'ARCH_GAUSSIAN_FHINC', 'ARCH_GAUSSIAN_FHINC', + 'DOIAU', 'OCNRES', 'ICERES', 'NUM_SND_COLLECTIVES', 'FHOUT_WAV', + 'FHOUT_HF_WAV', 'FHMAX_WAV', 'FHMAX_HF_WAV', 'FHMAX_WAV_GFS', + 'restart_interval_gdas', 'restart_interval_gfs', + 'DO_AERO_ANL', 'DO_AERO_FCST', 'DO_CA', 'DOIBP_WAV', 'DO_JEDIOCNVAR', 'DOHYBVAR_OCN', + 'NMEM_ENS', 'DO_JEDIATMVAR', 'DO_VRFY_OCEANDA', 'FHMAX_FITS', 'waveGRD', + 'IAUFHRS', 'DO_FIT2OBS', 'NET', 'FHOUT_HF_GFS', 'FHMAX_HF_GFS', 'REPLAY_ICS', + 'OFFSET_START_HOUR', 'ARCH_EXPDIR', 'EXPDIR', 'ARCH_EXPDIR_FREQ', 'ARCH_HASHES', + 'ARCH_DIFFS', 'SDATE', 'EDATE', 'HOMEgfs', 'DO_GEMPAK'] + + archive_dict = AttrDict() + for key in keys: + try: + archive_dict[key] = archive.task_config[key] + except KeyError as ee: + logger.warning(f"WARNING: key ({key}) not found in archive.task_config!") + + # Also import all COMIN* and COMOUT* directory and template variables + for key in archive.task_config.keys(): + if key.startswith(("COM_", "COMIN_", "COMOUT_")): + archive_dict[key] = archive.task_config.get(key) + + with chdir(config.ROTDIR): + + # Determine which archives to create + arcdir_set, atardir_sets = archive.configure(archive_dict) + + # Populate the product archive (ARCDIR) + archive.execute_store_products(arcdir_set) + + # Create the backup tarballs and store in ATARDIR + for atardir_set in atardir_sets: + archive.execute_backup_dataset(atardir_set) + + # Clean up any temporary files + archive.clean() + + +if __name__ == '__main__': + main() diff --git a/workflow/applications/applications.py b/workflow/applications/applications.py index 22e299df20..1acd20c527 100644 --- a/workflow/applications/applications.py +++ b/workflow/applications/applications.py @@ -90,6 +90,7 @@ def _get_run_options(self, conf: Configuration) -> Dict[str, Any]: run_options[run]['do_goes'] = run_base.get('DO_GOES', False) run_options[run]['do_mos'] = run_base.get('DO_MOS', False) run_options[run]['do_extractvars'] = run_base.get('DO_EXTRACTVARS', False) + run_options[run]['do_archtar'] = run_base.get('DO_ARCHTAR', False) run_options[run]['do_atm'] = run_base.get('DO_ATM', True) run_options[run]['do_wave'] = run_base.get('DO_WAVE', False) diff --git a/workflow/applications/gefs.py b/workflow/applications/gefs.py index 2e08ddc21d..81b12c70f2 100644 --- a/workflow/applications/gefs.py +++ b/workflow/applications/gefs.py @@ -92,7 +92,10 @@ def get_task_names(self): if options['do_extractvars']: tasks += ['extractvars'] + + if options['do_archtar']: + tasks += ['arch_tar'] - tasks += ['arch', 'cleanup'] + tasks += ['arch_vrfy', 'cleanup'] return {f"{self.run}": tasks} diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index f1b1cd1ea2..d7aebf7266 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -541,7 +541,7 @@ def extractvars(self): return task - def arch(self): + def arch_vrfy(self): deps = [] dep_dict = {'type': 'metatask', 'name': 'gefs_atmos_prod'} deps.append(rocoto.add_dependency(dep_dict)) @@ -568,8 +568,8 @@ def arch(self): deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps, dep_condition='and') - resources = self.get_resource('arch') - task_name = 'gefs_arch' + resources = self.get_resource('arch_vrfy') + task_name = 'gefs_arch_vrfy' task_dict = {'task_name': task_name, 'resources': resources, 'envars': self.envars, @@ -585,6 +585,50 @@ def arch(self): return task + def arch_tars(self): + deps = [] + dep_dict = {'type': 'metatask', 'name': 'gefs_atmos_prod'} + deps.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'metatask', 'name': 'gefs_atmos_ensstat'} + deps.append(rocoto.add_dependency(dep_dict)) + if self.options['do_ice']: + dep_dict = {'type': 'metatask', 'name': 'gefs_ice_prod'} + deps.append(rocoto.add_dependency(dep_dict)) + if self.options['do_ocean']: + dep_dict = {'type': 'metatask', 'name': 'gefs_ocean_prod'} + deps.append(rocoto.add_dependency(dep_dict)) + if self.options['do_wave']: + dep_dict = {'type': 'metatask', 'name': 'gefs_wave_post_grid'} + deps.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'metatask', 'name': 'gefs_wave_post_pnt'} + deps.append(rocoto.add_dependency(dep_dict)) + if self.options['do_wave_bnd']: + dep_dict = {'type': 'metatask', 'name': 'gefs_wave_post_bndpnt'} + deps.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'metatask', 'name': 'gefs_wave_post_bndpnt_bull'} + deps.append(rocoto.add_dependency(dep_dict)) + if self.options['do_extractvars']: + dep_dict = {'type': 'metatask', 'name': 'gefs_extractvars'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps, dep_condition='and') + + resources = self.get_resource('arch') + task_name = 'gefs_arch_tars' + task_dict = {'task_name': task_name, + 'resources': resources, + 'envars': self.envars, + 'cycledef': 'gefs', + 'dependency': dependencies, + 'command': f'{self.HOMEgfs}/jobs/rocoto/arch_tars.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' + } + + task = rocoto.create_task(task_dict) + + return task + def cleanup(self): deps = [] dep_dict = {'type': 'task', 'name': 'gefs_arch'} diff --git a/workflow/rocoto/tasks.py b/workflow/rocoto/tasks.py index 3c215414b5..d82e875753 100644 --- a/workflow/rocoto/tasks.py +++ b/workflow/rocoto/tasks.py @@ -11,9 +11,9 @@ class Tasks: - SERVICE_TASKS = ['arch', 'earc', 'stage_ic', 'cleanup'] + SERVICE_TASKS = ['arch_vrfy', 'arch_tars', 'earc', 'stage_ic', 'cleanup'] VALID_TASKS = ['aerosol_init', 'stage_ic', - 'prep', 'anal', 'sfcanl', 'analcalc', 'analdiag', 'arch', "cleanup", + 'prep', 'anal', 'sfcanl', 'analcalc', 'analdiag', 'arch_vrfy', 'arch_tars', "cleanup", 'prepatmiodaobs', 'atmanlinit', 'atmanlvar', 'atmanlfv3inc', 'atmanlfinal', 'prepoceanobs', 'marineanlinit', 'marineanlletkf', 'marinebmat', 'marineanlvar', 'ocnanalecen', 'marineanlchkpt', 'marineanlfinal', 'ocnanalvrfy', From 183a303955d60ab30e51c6fc0f1d0055aab4cb1a Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Fri, 17 Jan 2025 22:43:23 -0500 Subject: [PATCH 002/105] modified workflow\applications\gefs.py --- workflow/applications/gefs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/applications/gefs.py b/workflow/applications/gefs.py index 81b12c70f2..4a2d7e6df0 100644 --- a/workflow/applications/gefs.py +++ b/workflow/applications/gefs.py @@ -94,7 +94,7 @@ def get_task_names(self): tasks += ['extractvars'] if options['do_archtar']: - tasks += ['arch_tar'] + tasks += ['arch_tars'] tasks += ['arch_vrfy', 'cleanup'] From c585fe1b93380c658796fa2b3d58856f671bc263 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Fri, 17 Jan 2025 22:50:54 -0500 Subject: [PATCH 003/105] modified arch_vrfy var --- scripts/exglobal_archive_tars.py | 2 +- scripts/exglobal_archive_vrfy.py | 2 +- workflow/applications/gefs.py | 8 +++++++- workflow/rocoto/gefs_tasks.py | 2 +- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/scripts/exglobal_archive_tars.py b/scripts/exglobal_archive_tars.py index df49704e06..b8040af0d0 100644 --- a/scripts/exglobal_archive_tars.py +++ b/scripts/exglobal_archive_tars.py @@ -35,7 +35,7 @@ def main(): 'ARCH_GAUSSIAN_FHMAX', 'ARCH_GAUSSIAN_FHINC', 'ARCH_GAUSSIAN_FHINC', 'DOIAU', 'OCNRES', 'ICERES', 'NUM_SND_COLLECTIVES', 'FHOUT_WAV', 'FHOUT_HF_WAV', 'FHMAX_WAV', 'FHMAX_HF_WAV', 'FHMAX_WAV_GFS', - 'restart_interval_gdas', 'restart_interval_gfs', + 'restart_interval_gdas', 'restart_interval_gfs', 'DO_ARCHTAR', 'DO_AERO_ANL', 'DO_AERO_FCST', 'DO_CA', 'DOIBP_WAV', 'DO_JEDIOCNVAR', 'DOHYBVAR_OCN', 'NMEM_ENS', 'DO_JEDIATMVAR', 'DO_VRFY_OCEANDA', 'FHMAX_FITS', 'waveGRD', 'IAUFHRS', 'DO_FIT2OBS', 'NET', 'FHOUT_HF_GFS', 'FHMAX_HF_GFS', 'REPLAY_ICS', diff --git a/scripts/exglobal_archive_vrfy.py b/scripts/exglobal_archive_vrfy.py index df49704e06..b8040af0d0 100644 --- a/scripts/exglobal_archive_vrfy.py +++ b/scripts/exglobal_archive_vrfy.py @@ -35,7 +35,7 @@ def main(): 'ARCH_GAUSSIAN_FHMAX', 'ARCH_GAUSSIAN_FHINC', 'ARCH_GAUSSIAN_FHINC', 'DOIAU', 'OCNRES', 'ICERES', 'NUM_SND_COLLECTIVES', 'FHOUT_WAV', 'FHOUT_HF_WAV', 'FHMAX_WAV', 'FHMAX_HF_WAV', 'FHMAX_WAV_GFS', - 'restart_interval_gdas', 'restart_interval_gfs', + 'restart_interval_gdas', 'restart_interval_gfs', 'DO_ARCHTAR', 'DO_AERO_ANL', 'DO_AERO_FCST', 'DO_CA', 'DOIBP_WAV', 'DO_JEDIOCNVAR', 'DOHYBVAR_OCN', 'NMEM_ENS', 'DO_JEDIATMVAR', 'DO_VRFY_OCEANDA', 'FHMAX_FITS', 'waveGRD', 'IAUFHRS', 'DO_FIT2OBS', 'NET', 'FHOUT_HF_GFS', 'FHMAX_HF_GFS', 'REPLAY_ICS', diff --git a/workflow/applications/gefs.py b/workflow/applications/gefs.py index 4a2d7e6df0..82dc657fc2 100644 --- a/workflow/applications/gefs.py +++ b/workflow/applications/gefs.py @@ -28,7 +28,7 @@ def _get_app_configs(self, run): Returns the config_files that are involved in gefs """ options = self.run_options[run] - configs = ['stage_ic', 'fcst', 'atmos_products', 'arch', 'cleanup'] + configs = ['stage_ic', 'fcst', 'atmos_products', 'arch_vrfy', 'cleanup'] if options['nens'] > 0: configs += ['efcs', 'atmos_ensstat'] @@ -47,6 +47,12 @@ def _get_app_configs(self, run): if options['do_extractvars']: configs += ['extractvars'] + if options['do_extractvars']: + configs += ['extractvars'] + + if options['do_archtar']: + configs += ['arch_tars'] + return configs @staticmethod diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index d7aebf7266..c383a8a32f 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -612,7 +612,7 @@ def arch_tars(self): deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps, dep_condition='and') - resources = self.get_resource('arch') + resources = self.get_resource('arch_tars') task_name = 'gefs_arch_tars' task_dict = {'task_name': task_name, 'resources': resources, From 8ac2162c60abb09b24d1b498adfba63e9c76d5c6 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Fri, 17 Jan 2025 22:58:35 -0500 Subject: [PATCH 004/105] modified arch_vrfy vars in gefs config files --- parm/config/gefs/config.arch_tars | 15 +++++++++++++++ parm/config/gefs/config.arch_vrfy | 15 +++++++++++++++ parm/config/gefs/config.resources | 10 +++++++++- 3 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 parm/config/gefs/config.arch_tars create mode 100644 parm/config/gefs/config.arch_vrfy diff --git a/parm/config/gefs/config.arch_tars b/parm/config/gefs/config.arch_tars new file mode 100644 index 0000000000..6abaa40321 --- /dev/null +++ b/parm/config/gefs/config.arch_tars @@ -0,0 +1,15 @@ +#! /usr/bin/env bash + +########## config.arch ########## +# Archive specific + +echo "BEGIN: config.arch_tars" + +# Get task specific resources +. "${EXPDIR}/config.resources" arch_tars + +export ARCH_GAUSSIAN="YES" +export ARCH_GAUSSIAN_FHMAX=${FHMAX_GFS} +export ARCH_GAUSSIAN_FHINC=${FHOUT_GFS} + +echo "END: config.arch_tars" diff --git a/parm/config/gefs/config.arch_vrfy b/parm/config/gefs/config.arch_vrfy new file mode 100644 index 0000000000..616e5d2fff --- /dev/null +++ b/parm/config/gefs/config.arch_vrfy @@ -0,0 +1,15 @@ +#! /usr/bin/env bash + +########## config.vrfy ########## +# Archive specific + +echo "BEGIN: config.arch_vrfy" + +# Get task specific resources +. "${EXPDIR}/config.resources" arch_vrfy + +export ARCH_GAUSSIAN="YES" +export ARCH_GAUSSIAN_FHMAX=${FHMAX_GFS} +export ARCH_GAUSSIAN_FHINC=${FHOUT_GFS} + +echo "END: config.arch_vrfy" diff --git a/parm/config/gefs/config.resources b/parm/config/gefs/config.resources index bb33f3eb02..4102f3e4fe 100644 --- a/parm/config/gefs/config.resources +++ b/parm/config/gefs/config.resources @@ -323,7 +323,15 @@ case ${step} in export is_exclusive=False ;; - "arch") + "arch_vrfy") + export walltime="06:00:00" + export ntasks=1 + export tasks_per_node=1 + export threads_per_task=1 + export memory="4096M" + ;; + + "arch_tars") export walltime="06:00:00" export ntasks=1 export tasks_per_node=1 From b90347082f7edc591939fa7e09c4db054d70e9c0 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Fri, 17 Jan 2025 22:59:02 -0500 Subject: [PATCH 005/105] modified config files --- parm/config/gefs/config.arch | 15 --------------- 1 file changed, 15 deletions(-) delete mode 100644 parm/config/gefs/config.arch diff --git a/parm/config/gefs/config.arch b/parm/config/gefs/config.arch deleted file mode 100644 index a23bcce6ae..0000000000 --- a/parm/config/gefs/config.arch +++ /dev/null @@ -1,15 +0,0 @@ -#! /usr/bin/env bash - -########## config.arch ########## -# Archive specific - -echo "BEGIN: config.arch" - -# Get task specific resources -. "${EXPDIR}/config.resources" arch - -export ARCH_GAUSSIAN="YES" -export ARCH_GAUSSIAN_FHMAX=${FHMAX_GFS} -export ARCH_GAUSSIAN_FHINC=${FHOUT_GFS} - -echo "END: config.arch" From c74910d316167425fd63b8e20dbc73eb5f7bc75c Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Fri, 17 Jan 2025 23:16:21 -0500 Subject: [PATCH 006/105] modified exglobal_archive_vrfy.py --- jobs/JGLOBAL_ARCHIVE_TARS | 2 +- jobs/JGLOBAL_ARCHIVE_VRFY | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/jobs/JGLOBAL_ARCHIVE_TARS b/jobs/JGLOBAL_ARCHIVE_TARS index 96a9b863bc..d1b412c7f7 100644 --- a/jobs/JGLOBAL_ARCHIVE_TARS +++ b/jobs/JGLOBAL_ARCHIVE_TARS @@ -54,7 +54,7 @@ done # Run archive script ############################################################### -${GLOBALARCHIVESH:-${SCRgfs}/exglobal_archive.py} +${GLOBALARCHIVESH:-${SCRgfs}/exglobal_archive_tars.py} status=$? [[ ${status} -ne 0 ]] && exit "${status}" diff --git a/jobs/JGLOBAL_ARCHIVE_VRFY b/jobs/JGLOBAL_ARCHIVE_VRFY index 7fab09f646..b27b765328 100644 --- a/jobs/JGLOBAL_ARCHIVE_VRFY +++ b/jobs/JGLOBAL_ARCHIVE_VRFY @@ -54,7 +54,7 @@ done # Run archive script ############################################################### -${GLOBALARCHIVESH:-${SCRgfs}/exglobal_archive.py} +${GLOBALARCHIVESH:-${SCRgfs}/exglobal_archive_vrfy.py} status=$? [[ ${status} -ne 0 ]] && exit "${status}" From c9d414d41c697cbf35c93b03039849b0bf211eda Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Fri, 17 Jan 2025 23:36:56 -0500 Subject: [PATCH 007/105] modified defaults and JJOB --- jobs/JGLOBAL_ARCHIVE_TARS | 1 - parm/config/gefs/yaml/defaults.yaml | 1 - 2 files changed, 2 deletions(-) diff --git a/jobs/JGLOBAL_ARCHIVE_TARS b/jobs/JGLOBAL_ARCHIVE_TARS index d1b412c7f7..9ac6162eda 100644 --- a/jobs/JGLOBAL_ARCHIVE_TARS +++ b/jobs/JGLOBAL_ARCHIVE_TARS @@ -3,7 +3,6 @@ source "${HOMEgfs}/ush/preamble.sh" source "${HOMEgfs}/ush/jjob_header.sh" -e "arch_tars" -c "base arch_tars" - ############################################## # Set variables used in the script ############################################## diff --git a/parm/config/gefs/yaml/defaults.yaml b/parm/config/gefs/yaml/defaults.yaml index 9347ecad3e..df12b16282 100644 --- a/parm/config/gefs/yaml/defaults.yaml +++ b/parm/config/gefs/yaml/defaults.yaml @@ -19,7 +19,6 @@ base: FHOUT_ICE_GFS: 6 HPSSARCH: "NO" LOCALARCH: "NO" - VRFYARCH: "YES" SFS_POST: "NO" USE_OCN_ENS_PERTURB_FILES: "NO" USE_ATM_ENS_PERTURB_FILES: "NO" From 7d42e3648cc3ee4c4b6e530212fca87abefb4d5b Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Fri, 17 Jan 2025 23:40:25 -0500 Subject: [PATCH 008/105] modified jobs\JGLOBAL_ARCHIVE_TARS --- jobs/JGLOBAL_ARCHIVE_TARS | 72 +++++++++++++++++++-------------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/jobs/JGLOBAL_ARCHIVE_TARS b/jobs/JGLOBAL_ARCHIVE_TARS index 9ac6162eda..e1ee484aac 100644 --- a/jobs/JGLOBAL_ARCHIVE_TARS +++ b/jobs/JGLOBAL_ARCHIVE_TARS @@ -7,42 +7,42 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "arch_tars" -c "base arch_tars" # Set variables used in the script ############################################## YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ - COMIN_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL \ - COMIN_ATMOS_BUFR:COM_ATMOS_BUFR_TMPL \ - COMIN_ATMOS_GEMPAK:COM_ATMOS_GEMPAK_TMPL \ - COMIN_ATMOS_GENESIS:COM_ATMOS_GENESIS_TMPL \ - COMIN_ATMOS_HISTORY:COM_ATMOS_HISTORY_TMPL \ - COMIN_ATMOS_INPUT:COM_ATMOS_INPUT_TMPL \ - COMIN_ATMOS_MASTER:COM_ATMOS_MASTER_TMPL \ - COMIN_ATMOS_RESTART:COM_ATMOS_RESTART_TMPL \ - COMIN_ATMOS_TRACK:COM_ATMOS_TRACK_TMPL \ - COMIN_ATMOS_WMO:COM_ATMOS_WMO_TMPL \ - COMIN_CHEM_HISTORY:COM_CHEM_HISTORY_TMPL \ - COMIN_CHEM_ANALYSIS:COM_CHEM_ANALYSIS_TMPL \ - COMIN_MED_RESTART:COM_MED_RESTART_TMPL \ - COMIN_SNOW_ANALYSIS:COM_SNOW_ANALYSIS_TMPL \ - COMIN_ICE_HISTORY:COM_ICE_HISTORY_TMPL \ - COMIN_ICE_INPUT:COM_ICE_INPUT_TMPL \ - COMIN_ICE_RESTART:COM_ICE_RESTART_TMPL \ - COMIN_ICE_GRIB:COM_ICE_GRIB_TMPL \ - COMIN_OBS:COM_OBS_TMPL \ - COMIN_TOP:COM_TOP_TMPL \ - COMIN_OCEAN_HISTORY:COM_OCEAN_HISTORY_TMPL \ - COMIN_OCEAN_RESTART:COM_OCEAN_RESTART_TMPL \ - COMIN_OCEAN_GRIB:COM_OCEAN_GRIB_TMPL \ - COMIN_OCEAN_NETCDF:COM_OCEAN_NETCDF_TMPL \ - COMIN_OCEAN_ANALYSIS:COM_OCEAN_ANALYSIS_TMPL \ - COMIN_OCEAN_BMATRIX:COM_OCEAN_BMATRIX_TMPL \ - COMIN_ICE_BMATRIX:COM_ICE_BMATRIX_TMPL \ - COMIN_WAVE_GRID:COM_WAVE_GRID_TMPL \ - COMIN_WAVE_HISTORY:COM_WAVE_HISTORY_TMPL \ - COMIN_WAVE_STATION:COM_WAVE_STATION_TMPL \ - COMIN_WAVE_RESTART:COM_WAVE_RESTART_TMPL \ - COMIN_ATMOS_OZNMON:COM_ATMOS_OZNMON_TMPL \ - COMIN_ATMOS_RADMON:COM_ATMOS_RADMON_TMPL \ - COMIN_ATMOS_MINMON:COM_ATMOS_MINMON_TMPL \ - COMIN_CONF:COM_CONF_TMPL \ - COMOUT_ATMOS_TRACK:COM_ATMOS_TRACK_TMPL + COMIN_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL \ + COMIN_ATMOS_BUFR:COM_ATMOS_BUFR_TMPL \ + COMIN_ATMOS_GEMPAK:COM_ATMOS_GEMPAK_TMPL \ + COMIN_ATMOS_GENESIS:COM_ATMOS_GENESIS_TMPL \ + COMIN_ATMOS_HISTORY:COM_ATMOS_HISTORY_TMPL \ + COMIN_ATMOS_INPUT:COM_ATMOS_INPUT_TMPL \ + COMIN_ATMOS_MASTER:COM_ATMOS_MASTER_TMPL \ + COMIN_ATMOS_RESTART:COM_ATMOS_RESTART_TMPL \ + COMIN_ATMOS_TRACK:COM_ATMOS_TRACK_TMPL \ + COMIN_ATMOS_WMO:COM_ATMOS_WMO_TMPL \ + COMIN_CHEM_HISTORY:COM_CHEM_HISTORY_TMPL \ + COMIN_CHEM_ANALYSIS:COM_CHEM_ANALYSIS_TMPL \ + COMIN_MED_RESTART:COM_MED_RESTART_TMPL \ + COMIN_SNOW_ANALYSIS:COM_SNOW_ANALYSIS_TMPL \ + COMIN_ICE_HISTORY:COM_ICE_HISTORY_TMPL \ + COMIN_ICE_INPUT:COM_ICE_INPUT_TMPL \ + COMIN_ICE_RESTART:COM_ICE_RESTART_TMPL \ + COMIN_ICE_GRIB:COM_ICE_GRIB_TMPL \ + COMIN_OBS:COM_OBS_TMPL \ + COMIN_TOP:COM_TOP_TMPL \ + COMIN_OCEAN_HISTORY:COM_OCEAN_HISTORY_TMPL \ + COMIN_OCEAN_RESTART:COM_OCEAN_RESTART_TMPL \ + COMIN_OCEAN_GRIB:COM_OCEAN_GRIB_TMPL \ + COMIN_OCEAN_NETCDF:COM_OCEAN_NETCDF_TMPL \ + COMIN_OCEAN_ANALYSIS:COM_OCEAN_ANALYSIS_TMPL \ + COMIN_OCEAN_BMATRIX:COM_OCEAN_BMATRIX_TMPL \ + COMIN_ICE_BMATRIX:COM_ICE_BMATRIX_TMPL \ + COMIN_WAVE_GRID:COM_WAVE_GRID_TMPL \ + COMIN_WAVE_HISTORY:COM_WAVE_HISTORY_TMPL \ + COMIN_WAVE_STATION:COM_WAVE_STATION_TMPL \ + COMIN_WAVE_RESTART:COM_WAVE_RESTART_TMPL \ + COMIN_ATMOS_OZNMON:COM_ATMOS_OZNMON_TMPL \ + COMIN_ATMOS_RADMON:COM_ATMOS_RADMON_TMPL \ + COMIN_ATMOS_MINMON:COM_ATMOS_MINMON_TMPL \ + COMIN_CONF:COM_CONF_TMPL \ + COMOUT_ATMOS_TRACK:COM_ATMOS_TRACK_TMPL for grid in "0p25" "0p50" "1p00"; do YMD=${PDY} HH=${cyc} GRID=${grid} declare_from_tmpl -rx \ From fc69dcd4ce7b2792de8bd1273b17f0cd83de15f9 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Fri, 17 Jan 2025 23:41:18 -0500 Subject: [PATCH 009/105] modified jobs\JGLOBAL_ARCHIVE_TARS --- jobs/JGLOBAL_ARCHIVE_TARS | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jobs/JGLOBAL_ARCHIVE_TARS b/jobs/JGLOBAL_ARCHIVE_TARS index e1ee484aac..784acdfad1 100644 --- a/jobs/JGLOBAL_ARCHIVE_TARS +++ b/jobs/JGLOBAL_ARCHIVE_TARS @@ -32,8 +32,8 @@ YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ COMIN_OCEAN_GRIB:COM_OCEAN_GRIB_TMPL \ COMIN_OCEAN_NETCDF:COM_OCEAN_NETCDF_TMPL \ COMIN_OCEAN_ANALYSIS:COM_OCEAN_ANALYSIS_TMPL \ - COMIN_OCEAN_BMATRIX:COM_OCEAN_BMATRIX_TMPL \ - COMIN_ICE_BMATRIX:COM_ICE_BMATRIX_TMPL \ + COMIN_OCEAN_BMATRIX:COM_OCEAN_BMATRIX_TMPL \ + COMIN_ICE_BMATRIX:COM_ICE_BMATRIX_TMPL \ COMIN_WAVE_GRID:COM_WAVE_GRID_TMPL \ COMIN_WAVE_HISTORY:COM_WAVE_HISTORY_TMPL \ COMIN_WAVE_STATION:COM_WAVE_STATION_TMPL \ From 8e523a1c5f4bc3d311897c3867531ff10e9f4acc Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Fri, 17 Jan 2025 23:43:53 -0500 Subject: [PATCH 010/105] modified config files --- parm/config/gefs/config.arch_tars | 2 +- parm/config/gefs/config.arch_vrfy | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/parm/config/gefs/config.arch_tars b/parm/config/gefs/config.arch_tars index 6abaa40321..7605cc04c1 100644 --- a/parm/config/gefs/config.arch_tars +++ b/parm/config/gefs/config.arch_tars @@ -1,6 +1,6 @@ #! /usr/bin/env bash -########## config.arch ########## +########## config.arch_tars ########## # Archive specific echo "BEGIN: config.arch_tars" diff --git a/parm/config/gefs/config.arch_vrfy b/parm/config/gefs/config.arch_vrfy index 616e5d2fff..cb668a48e2 100644 --- a/parm/config/gefs/config.arch_vrfy +++ b/parm/config/gefs/config.arch_vrfy @@ -1,6 +1,6 @@ #! /usr/bin/env bash -########## config.vrfy ########## +########## config.arch_vrfy ########## # Archive specific echo "BEGIN: config.arch_vrfy" From 5cafe8a749c48359f124261b6971f38b1ec9e660 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Fri, 17 Jan 2025 23:49:46 -0500 Subject: [PATCH 011/105] modified parm\config\gefs\config.base --- parm/config/gefs/config.base | 2 ++ 1 file changed, 2 insertions(+) diff --git a/parm/config/gefs/config.base b/parm/config/gefs/config.base index 884afbf89e..db6e4bdaed 100644 --- a/parm/config/gefs/config.base +++ b/parm/config/gefs/config.base @@ -341,6 +341,8 @@ if [[ ${HPSSARCH} = "YES" ]] && [[ ${LOCALARCH} = "YES" ]]; then exit 3 elif [[ ${HPSSARCH} = "YES" ]] || [[ ${LOCALARCH} = "YES" ]]; then export DO_ARCHTAR="YES" +else + export DO_ARCHTAR="NO" fi export ARCH_CYC=00 # Archive data at this cycle for warm start and/or forecast-only capabilities export ARCH_WARMICFREQ=4 # Archive frequency in days for warm start capability From 3b2074fc37d1399cd4e3441cc34d9b5d9e3c18b6 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Sat, 18 Jan 2025 11:53:15 -0500 Subject: [PATCH 012/105] modify jjobs --- jobs/JGLOBAL_ARCHIVE_TARS | 1 - jobs/JGLOBAL_ARCHIVE_VRFY | 1 - 2 files changed, 2 deletions(-) diff --git a/jobs/JGLOBAL_ARCHIVE_TARS b/jobs/JGLOBAL_ARCHIVE_TARS index 784acdfad1..5573a6c778 100644 --- a/jobs/JGLOBAL_ARCHIVE_TARS +++ b/jobs/JGLOBAL_ARCHIVE_TARS @@ -68,7 +68,6 @@ if [[ -e "${pgmout}" ]] ; then cat "${pgmout}" fi - ########################################## # Remove the Temporary working directory ########################################## diff --git a/jobs/JGLOBAL_ARCHIVE_VRFY b/jobs/JGLOBAL_ARCHIVE_VRFY index b27b765328..aba9edf06f 100644 --- a/jobs/JGLOBAL_ARCHIVE_VRFY +++ b/jobs/JGLOBAL_ARCHIVE_VRFY @@ -69,7 +69,6 @@ if [[ -e "${pgmout}" ]] ; then cat "${pgmout}" fi - ########################################## # Remove the Temporary working directory ########################################## From 3008b33dd554b290bdd4c6c45490adc6e9e8ed76 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Sat, 18 Jan 2025 12:00:34 -0500 Subject: [PATCH 013/105] correct a typo --- workflow/applications/gefs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/workflow/applications/gefs.py b/workflow/applications/gefs.py index 82dc657fc2..5329c88b9e 100644 --- a/workflow/applications/gefs.py +++ b/workflow/applications/gefs.py @@ -98,9 +98,9 @@ def get_task_names(self): if options['do_extractvars']: tasks += ['extractvars'] - + if options['do_archtar']: - tasks += ['arch_tars'] + tasks += ['arch_tars'] tasks += ['arch_vrfy', 'cleanup'] From 4c145ca00c6eea8dbfe5c2fd5f1d210b485323a7 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Sat, 18 Jan 2025 12:09:29 -0500 Subject: [PATCH 014/105] modified gefs.py --- workflow/applications/gefs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/applications/gefs.py b/workflow/applications/gefs.py index 5329c88b9e..fbf15cc3d7 100644 --- a/workflow/applications/gefs.py +++ b/workflow/applications/gefs.py @@ -102,6 +102,6 @@ def get_task_names(self): if options['do_archtar']: tasks += ['arch_tars'] - tasks += ['arch_vrfy', 'cleanup'] + tasks += ['arch_vrfy', 'cleanup'] return {f"{self.run}": tasks} From cdd5acf9fa724f19a449ca1d8867832b9edcc79a Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Sat, 18 Jan 2025 12:10:33 -0500 Subject: [PATCH 015/105] modified workflow\applications\gefs.py --- workflow/applications/gefs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/applications/gefs.py b/workflow/applications/gefs.py index fbf15cc3d7..5329c88b9e 100644 --- a/workflow/applications/gefs.py +++ b/workflow/applications/gefs.py @@ -102,6 +102,6 @@ def get_task_names(self): if options['do_archtar']: tasks += ['arch_tars'] - tasks += ['arch_vrfy', 'cleanup'] + tasks += ['arch_vrfy', 'cleanup'] return {f"{self.run}": tasks} From 5c9b3f560f35da63070f3f82b8ad95b861b3efec Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Sat, 18 Jan 2025 12:23:00 -0500 Subject: [PATCH 016/105] modified gefs.py --- workflow/applications/gefs.py | 102 ---------------------------------- 1 file changed, 102 deletions(-) diff --git a/workflow/applications/gefs.py b/workflow/applications/gefs.py index 5329c88b9e..f1e1319492 100644 --- a/workflow/applications/gefs.py +++ b/workflow/applications/gefs.py @@ -3,105 +3,3 @@ from wxflow import Configuration -class GEFSAppConfig(AppConfig): - ''' - Class to define GEFS configurations - ''' - - def __init__(self, conf: Configuration): - super().__init__(conf) - - base = conf.parse_config('config.base') - self.run = base.get('RUN', 'gefs') - self.runs = [self.run] - - def _get_run_options(self, conf: Configuration) -> Dict[str, Any]: - - run_options = super()._get_run_options(conf) - - run_options[self.run]['nens'] = conf.parse_config('config.base').get('NMEM_ENS', 0) - - return run_options - - def _get_app_configs(self, run): - """ - Returns the config_files that are involved in gefs - """ - options = self.run_options[run] - configs = ['stage_ic', 'fcst', 'atmos_products', 'arch_vrfy', 'cleanup'] - - if options['nens'] > 0: - configs += ['efcs', 'atmos_ensstat'] - - if options['do_wave']: - configs += ['waveinit', 'wavepostsbs', 'wavepostpnt'] - if options['do_wave_bnd']: - configs += ['wavepostbndpnt', 'wavepostbndpntbll'] - - if options['do_ocean'] or options['do_ice']: - configs += ['oceanice_products'] - - if options['do_aero_fcst']: - configs += ['prep_emissions'] - - if options['do_extractvars']: - configs += ['extractvars'] - - if options['do_extractvars']: - configs += ['extractvars'] - - if options['do_archtar']: - configs += ['arch_tars'] - - return configs - - @staticmethod - def _update_base(base_in): - - base_out = base_in.copy() - base_out['RUN'] = 'gefs' - - return base_out - - def get_task_names(self): - - options = self.run_options[self.run] - tasks = ['stage_ic'] - - if options['do_wave']: - tasks += ['waveinit'] - - if options['do_aero_fcst']: - tasks += ['prep_emissions'] - - tasks += ['fcst'] - - if options['nens'] > 0: - tasks += ['efcs'] - - tasks += ['atmos_prod'] - - if options['nens'] > 0: - tasks += ['atmos_ensstat'] - - if options['do_ocean']: - tasks += ['ocean_prod'] - - if options['do_ice']: - tasks += ['ice_prod'] - - if options['do_wave']: - tasks += ['wavepostsbs'] - if options['do_wave_bnd']: - tasks += ['wavepostbndpnt', 'wavepostbndpntbll'] - tasks += ['wavepostpnt'] - - if options['do_extractvars']: - tasks += ['extractvars'] - - if options['do_archtar']: - tasks += ['arch_tars'] - - tasks += ['arch_vrfy', 'cleanup'] - - return {f"{self.run}": tasks} From 260354e15e4662f107078c5e3573e3ffcafc94de Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Sat, 18 Jan 2025 12:26:02 -0500 Subject: [PATCH 017/105] modified workflow\applications\gefs.py --- workflow/applications/gefs.py | 102 ++++++++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) diff --git a/workflow/applications/gefs.py b/workflow/applications/gefs.py index f1e1319492..5329c88b9e 100644 --- a/workflow/applications/gefs.py +++ b/workflow/applications/gefs.py @@ -3,3 +3,105 @@ from wxflow import Configuration +class GEFSAppConfig(AppConfig): + ''' + Class to define GEFS configurations + ''' + + def __init__(self, conf: Configuration): + super().__init__(conf) + + base = conf.parse_config('config.base') + self.run = base.get('RUN', 'gefs') + self.runs = [self.run] + + def _get_run_options(self, conf: Configuration) -> Dict[str, Any]: + + run_options = super()._get_run_options(conf) + + run_options[self.run]['nens'] = conf.parse_config('config.base').get('NMEM_ENS', 0) + + return run_options + + def _get_app_configs(self, run): + """ + Returns the config_files that are involved in gefs + """ + options = self.run_options[run] + configs = ['stage_ic', 'fcst', 'atmos_products', 'arch_vrfy', 'cleanup'] + + if options['nens'] > 0: + configs += ['efcs', 'atmos_ensstat'] + + if options['do_wave']: + configs += ['waveinit', 'wavepostsbs', 'wavepostpnt'] + if options['do_wave_bnd']: + configs += ['wavepostbndpnt', 'wavepostbndpntbll'] + + if options['do_ocean'] or options['do_ice']: + configs += ['oceanice_products'] + + if options['do_aero_fcst']: + configs += ['prep_emissions'] + + if options['do_extractvars']: + configs += ['extractvars'] + + if options['do_extractvars']: + configs += ['extractvars'] + + if options['do_archtar']: + configs += ['arch_tars'] + + return configs + + @staticmethod + def _update_base(base_in): + + base_out = base_in.copy() + base_out['RUN'] = 'gefs' + + return base_out + + def get_task_names(self): + + options = self.run_options[self.run] + tasks = ['stage_ic'] + + if options['do_wave']: + tasks += ['waveinit'] + + if options['do_aero_fcst']: + tasks += ['prep_emissions'] + + tasks += ['fcst'] + + if options['nens'] > 0: + tasks += ['efcs'] + + tasks += ['atmos_prod'] + + if options['nens'] > 0: + tasks += ['atmos_ensstat'] + + if options['do_ocean']: + tasks += ['ocean_prod'] + + if options['do_ice']: + tasks += ['ice_prod'] + + if options['do_wave']: + tasks += ['wavepostsbs'] + if options['do_wave_bnd']: + tasks += ['wavepostbndpnt', 'wavepostbndpntbll'] + tasks += ['wavepostpnt'] + + if options['do_extractvars']: + tasks += ['extractvars'] + + if options['do_archtar']: + tasks += ['arch_tars'] + + tasks += ['arch_vrfy', 'cleanup'] + + return {f"{self.run}": tasks} From e880082c84766f4c920e1836b0e413dda0e6ec3e Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Sat, 18 Jan 2025 12:26:53 -0500 Subject: [PATCH 018/105] modifed gefs.py --- workflow/applications/gefs.py | 104 ---------------------------------- 1 file changed, 104 deletions(-) diff --git a/workflow/applications/gefs.py b/workflow/applications/gefs.py index 5329c88b9e..6f2e80a090 100644 --- a/workflow/applications/gefs.py +++ b/workflow/applications/gefs.py @@ -1,107 +1,3 @@ from applications.applications import AppConfig from typing import Dict, Any from wxflow import Configuration - - -class GEFSAppConfig(AppConfig): - ''' - Class to define GEFS configurations - ''' - - def __init__(self, conf: Configuration): - super().__init__(conf) - - base = conf.parse_config('config.base') - self.run = base.get('RUN', 'gefs') - self.runs = [self.run] - - def _get_run_options(self, conf: Configuration) -> Dict[str, Any]: - - run_options = super()._get_run_options(conf) - - run_options[self.run]['nens'] = conf.parse_config('config.base').get('NMEM_ENS', 0) - - return run_options - - def _get_app_configs(self, run): - """ - Returns the config_files that are involved in gefs - """ - options = self.run_options[run] - configs = ['stage_ic', 'fcst', 'atmos_products', 'arch_vrfy', 'cleanup'] - - if options['nens'] > 0: - configs += ['efcs', 'atmos_ensstat'] - - if options['do_wave']: - configs += ['waveinit', 'wavepostsbs', 'wavepostpnt'] - if options['do_wave_bnd']: - configs += ['wavepostbndpnt', 'wavepostbndpntbll'] - - if options['do_ocean'] or options['do_ice']: - configs += ['oceanice_products'] - - if options['do_aero_fcst']: - configs += ['prep_emissions'] - - if options['do_extractvars']: - configs += ['extractvars'] - - if options['do_extractvars']: - configs += ['extractvars'] - - if options['do_archtar']: - configs += ['arch_tars'] - - return configs - - @staticmethod - def _update_base(base_in): - - base_out = base_in.copy() - base_out['RUN'] = 'gefs' - - return base_out - - def get_task_names(self): - - options = self.run_options[self.run] - tasks = ['stage_ic'] - - if options['do_wave']: - tasks += ['waveinit'] - - if options['do_aero_fcst']: - tasks += ['prep_emissions'] - - tasks += ['fcst'] - - if options['nens'] > 0: - tasks += ['efcs'] - - tasks += ['atmos_prod'] - - if options['nens'] > 0: - tasks += ['atmos_ensstat'] - - if options['do_ocean']: - tasks += ['ocean_prod'] - - if options['do_ice']: - tasks += ['ice_prod'] - - if options['do_wave']: - tasks += ['wavepostsbs'] - if options['do_wave_bnd']: - tasks += ['wavepostbndpnt', 'wavepostbndpntbll'] - tasks += ['wavepostpnt'] - - if options['do_extractvars']: - tasks += ['extractvars'] - - if options['do_archtar']: - tasks += ['arch_tars'] - - tasks += ['arch_vrfy', 'cleanup'] - - return {f"{self.run}": tasks} From 4fab72f9346b55396df244e39850c92414e01a1f Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Sat, 18 Jan 2025 12:30:04 -0500 Subject: [PATCH 019/105] modified gefs.py --- workflow/applications/gefs.py | 99 +++++++++++++++++++++++++++++++++++ 1 file changed, 99 insertions(+) diff --git a/workflow/applications/gefs.py b/workflow/applications/gefs.py index 6f2e80a090..6e06461aa2 100644 --- a/workflow/applications/gefs.py +++ b/workflow/applications/gefs.py @@ -1,3 +1,102 @@ from applications.applications import AppConfig from typing import Dict, Any from wxflow import Configuration + + +class GEFSAppConfig(AppConfig): + ''' + Class to define GEFS configurations + ''' + + def __init__(self, conf: Configuration): + super().__init__(conf) + + base = conf.parse_config('config.base') + self.run = base.get('RUN', 'gefs') + self.runs = [self.run] + + def _get_run_options(self, conf: Configuration) -> Dict[str, Any]: + + run_options = super()._get_run_options(conf) + + run_options[self.run]['nens'] = conf.parse_config('config.base').get('NMEM_ENS', 0) + + return run_options + + def _get_app_configs(self, run): + """ + Returns the config_files that are involved in gefs + """ + options = self.run_options[run] + configs = ['stage_ic', 'fcst', 'atmos_products', 'arch_vrfy', 'cleanup'] + + if options['nens'] > 0: + configs += ['efcs', 'atmos_ensstat'] + + if options['do_wave']: + configs += ['waveinit', 'wavepostsbs', 'wavepostpnt'] + if options['do_wave_bnd']: + configs += ['wavepostbndpnt', 'wavepostbndpntbll'] + + if options['do_ocean'] or options['do_ice']: + configs += ['oceanice_products'] + + if options['do_aero_fcst']: + configs += ['prep_emissions'] + + if options['do_extractvars']: + configs += ['extractvars'] + + if options['do_extractvars']: + configs += ['extractvars'] + + if options['do_archtar']: + configs += ['arch_tars'] + + return configs + + @staticmethod + def _update_base(base_in): + + base_out = base_in.copy() + base_out['RUN'] = 'gefs' + + return base_out + + def get_task_names(self): + + options = self.run_options[self.run] + tasks = ['stage_ic'] + + if options['do_wave']: + tasks += ['waveinit'] + + if options['do_aero_fcst']: + tasks += ['prep_emissions'] + + tasks += ['fcst'] + + if options['nens'] > 0: + tasks += ['efcs'] + + tasks += ['atmos_prod'] + + if options['nens'] > 0: + tasks += ['atmos_ensstat'] + + if options['do_ocean']: + tasks += ['ocean_prod'] + + if options['do_ice']: + tasks += ['ice_prod'] + + if options['do_wave']: + tasks += ['wavepostsbs'] + if options['do_wave_bnd']: + tasks += ['wavepostbndpnt', 'wavepostbndpntbll'] + tasks += ['wavepostpnt'] + + if options['do_extractvars']: + tasks += ['extractvars'] + + return {f"{self.run}": tasks} From 9e54cc0f9bafa6421d6d6853aa38848033cc4b58 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Sat, 18 Jan 2025 12:35:48 -0500 Subject: [PATCH 020/105] modified workflow\applications\gefs.py --- workflow/applications/gefs.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/workflow/applications/gefs.py b/workflow/applications/gefs.py index 6e06461aa2..5329c88b9e 100644 --- a/workflow/applications/gefs.py +++ b/workflow/applications/gefs.py @@ -99,4 +99,9 @@ def get_task_names(self): if options['do_extractvars']: tasks += ['extractvars'] + if options['do_archtar']: + tasks += ['arch_tars'] + + tasks += ['arch_vrfy', 'cleanup'] + return {f"{self.run}": tasks} From 0f0c7c26815bbf3c1bf8588cc261ceacfb22796c Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Sat, 18 Jan 2025 12:38:27 -0500 Subject: [PATCH 021/105] modifed gefs.py --- workflow/applications/gefs.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/workflow/applications/gefs.py b/workflow/applications/gefs.py index 5329c88b9e..8489d44f48 100644 --- a/workflow/applications/gefs.py +++ b/workflow/applications/gefs.py @@ -2,7 +2,6 @@ from typing import Dict, Any from wxflow import Configuration - class GEFSAppConfig(AppConfig): ''' Class to define GEFS configurations @@ -49,7 +48,7 @@ def _get_app_configs(self, run): if options['do_extractvars']: configs += ['extractvars'] - + if options['do_archtar']: configs += ['arch_tars'] From 1b736974637019ab450be9ca6735518f0d482d67 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Sat, 18 Jan 2025 12:39:08 -0500 Subject: [PATCH 022/105] modifed workflow\applications\gefs.py --- workflow/applications/gefs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/workflow/applications/gefs.py b/workflow/applications/gefs.py index 8489d44f48..ad9472cca2 100644 --- a/workflow/applications/gefs.py +++ b/workflow/applications/gefs.py @@ -2,6 +2,7 @@ from typing import Dict, Any from wxflow import Configuration + class GEFSAppConfig(AppConfig): ''' Class to define GEFS configurations From 14fc7102d2695d7374e183433c11d0b7fcfca5a4 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Sat, 18 Jan 2025 12:55:36 -0500 Subject: [PATCH 023/105] modified workflow\rocoto\gefs_tasks.py --- workflow/rocoto/gefs_tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index c383a8a32f..ff452ed1d9 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -575,7 +575,7 @@ def arch_vrfy(self): 'envars': self.envars, 'cycledef': 'gefs', 'dependency': dependencies, - 'command': f'{self.HOMEgfs}/jobs/rocoto/arch.sh', + 'command': f'{self.HOMEgfs}/jobs/rocoto/arch.vrfysh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' From 4493271470a1c90cec4e3ae6fa3b7ed99858f519 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Sat, 18 Jan 2025 12:59:29 -0500 Subject: [PATCH 024/105] modifed workflow\rocoto\gefs_tasks.py --- workflow/rocoto/gefs_tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index ff452ed1d9..de57c5728c 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -575,7 +575,7 @@ def arch_vrfy(self): 'envars': self.envars, 'cycledef': 'gefs', 'dependency': dependencies, - 'command': f'{self.HOMEgfs}/jobs/rocoto/arch.vrfysh', + 'command': f'{self.HOMEgfs}/jobs/rocoto/arch_vrfy.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' From 4d6ad0c002362047d3ae1b945728824b90b2f347 Mon Sep 17 00:00:00 2001 From: "Anton.Fernando" Date: Sat, 18 Jan 2025 18:32:30 +0000 Subject: [PATCH 025/105] change permission --- jobs/JGLOBAL_ARCHIVE_TARS | 0 jobs/JGLOBAL_ARCHIVE_VRFY | 0 jobs/rocoto/arch_tars.sh | 0 jobs/rocoto/arch_vrfy.sh | 0 parm/config/gefs/config.arch_tars | 0 parm/config/gefs/config.arch_vrfy | 0 scripts/exglobal_archive_tars.py | 0 scripts/exglobal_archive_vrfy.py | 0 8 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 jobs/JGLOBAL_ARCHIVE_TARS mode change 100644 => 100755 jobs/JGLOBAL_ARCHIVE_VRFY mode change 100644 => 100755 jobs/rocoto/arch_tars.sh mode change 100644 => 100755 jobs/rocoto/arch_vrfy.sh mode change 100644 => 100755 parm/config/gefs/config.arch_tars mode change 100644 => 100755 parm/config/gefs/config.arch_vrfy mode change 100644 => 100755 scripts/exglobal_archive_tars.py mode change 100644 => 100755 scripts/exglobal_archive_vrfy.py diff --git a/jobs/JGLOBAL_ARCHIVE_TARS b/jobs/JGLOBAL_ARCHIVE_TARS old mode 100644 new mode 100755 diff --git a/jobs/JGLOBAL_ARCHIVE_VRFY b/jobs/JGLOBAL_ARCHIVE_VRFY old mode 100644 new mode 100755 diff --git a/jobs/rocoto/arch_tars.sh b/jobs/rocoto/arch_tars.sh old mode 100644 new mode 100755 diff --git a/jobs/rocoto/arch_vrfy.sh b/jobs/rocoto/arch_vrfy.sh old mode 100644 new mode 100755 diff --git a/parm/config/gefs/config.arch_tars b/parm/config/gefs/config.arch_tars old mode 100644 new mode 100755 diff --git a/parm/config/gefs/config.arch_vrfy b/parm/config/gefs/config.arch_vrfy old mode 100644 new mode 100755 diff --git a/scripts/exglobal_archive_tars.py b/scripts/exglobal_archive_tars.py old mode 100644 new mode 100755 diff --git a/scripts/exglobal_archive_vrfy.py b/scripts/exglobal_archive_vrfy.py old mode 100644 new mode 100755 From 8d97a783206bc77d2e9e0e556579966296b5691a Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Sat, 18 Jan 2025 19:27:09 -0500 Subject: [PATCH 026/105] modified archive.py functions --- scripts/exglobal_archive_tars.py | 5 +-- scripts/exglobal_archive_vrfy.py | 6 +--- ush/python/pygfs/task/archive.py | 54 ++++++++++++++++++++++++++------ 3 files changed, 47 insertions(+), 18 deletions(-) diff --git a/scripts/exglobal_archive_tars.py b/scripts/exglobal_archive_tars.py index b8040af0d0..fb3f0952d8 100644 --- a/scripts/exglobal_archive_tars.py +++ b/scripts/exglobal_archive_tars.py @@ -57,10 +57,7 @@ def main(): with chdir(config.ROTDIR): # Determine which archives to create - arcdir_set, atardir_sets = archive.configure(archive_dict) - - # Populate the product archive (ARCDIR) - archive.execute_store_products(arcdir_set) + atardir_sets = archive.configure_atardir(archive_dict) # Create the backup tarballs and store in ATARDIR for atardir_set in atardir_sets: diff --git a/scripts/exglobal_archive_vrfy.py b/scripts/exglobal_archive_vrfy.py index b8040af0d0..a1a613151e 100644 --- a/scripts/exglobal_archive_vrfy.py +++ b/scripts/exglobal_archive_vrfy.py @@ -57,15 +57,11 @@ def main(): with chdir(config.ROTDIR): # Determine which archives to create - arcdir_set, atardir_sets = archive.configure(archive_dict) + arcdir_set = archive.configure_arcdir(archive_dict) # Populate the product archive (ARCDIR) archive.execute_store_products(arcdir_set) - # Create the backup tarballs and store in ATARDIR - for atardir_set in atardir_sets: - archive.execute_backup_dataset(atardir_set) - # Clean up any temporary files archive.clean() diff --git a/ush/python/pygfs/task/archive.py b/ush/python/pygfs/task/archive.py index ed63a22230..9835fa7654 100644 --- a/ush/python/pygfs/task/archive.py +++ b/ush/python/pygfs/task/archive.py @@ -48,8 +48,8 @@ def __init__(self, config: Dict[str, Any]) -> None: self.archive_expdir = False @logit(logger) - def configure(self, arch_dict: Dict[str, Any]) -> (Dict[str, Any], List[Dict[str, Any]]): - """Determine which tarballs will need to be created. + def configure_arcdir(self, arch_dict: Dict[str, Any]) -> (Dict[str, Any]): + """Determine which files will need to be created to archive tp arcdir. Parameters ---------- @@ -60,8 +60,6 @@ def configure(self, arch_dict: Dict[str, Any]) -> (Dict[str, Any], List[Dict[str ------ arcdir_set : Dict[str, Any] Set of FileHandler instructions to copy files to the ARCDIR - atardir_sets : List[Dict[str, Any]] - List of tarballs and instructions for creating them via tar or htar """ if not os.path.isdir(arch_dict.ROTDIR): @@ -89,6 +87,45 @@ def configure(self, arch_dict: Dict[str, Any]) -> (Dict[str, Any], List[Dict[str arcdir_set = Archive._construct_arcdir_set(arcdir_j2yaml, arch_dict) + # Collect datasets that need to be archived + self.tar_cmd = "" + arcdir_set = (arcdir_set, []) + + return arcdir_set + + @logit(logger) + def configure_atardir(self, arch_dict: Dict[str, Any]) -> (List[Dict[str, Any]]): + """Determine which tarballs will need to be created. + + Parameters + ---------- + arch_dict : Dict[str, Any] + Task specific keys, e.g. runtime options (DO_AERO_FCST, DO_ICE, etc) + + Return + ------ + atardir_sets : List[Dict[str, Any]] + List of tarballs and instructions for creating them via tar or htar + """ + + if not os.path.isdir(arch_dict.ROTDIR): + raise FileNotFoundError(f"FATAL ERROR: The ROTDIR ({arch_dict.ROTDIR}) does not exist!") + + if arch_dict.RUN in ["gdas", "gfs"]: + + # Copy the cyclone track files and rename the experiments + # TODO This really doesn't belong in archiving and should be moved elsewhere + Archive._rename_cyclone_expt(arch_dict) + + archive_parm = os.path.join(arch_dict.PARMgfs, "archive") + + # Add the glob.glob function for capturing log filenames + # TODO remove this kludge once log filenames are explicit + arch_dict['glob'] = glob.glob + + # Add the os.path.exists function to the dict for yaml parsing + arch_dict['path_exists'] = os.path.exists + if not os.path.isdir(arch_dict.ROTDIR): raise FileNotFoundError(f"FATAL ERROR: The ROTDIR ({arch_dict.ROTDIR}) does not exist!") @@ -109,9 +146,8 @@ def configure(self, arch_dict: Dict[str, Any]) -> (Dict[str, Any], List[Dict[str self.chgrp_cmd = chgrp self.chmod_cmd = os.chmod self.rm_cmd = rm_p - else: # Only perform local archiving. Do not create tarballs. - self.tar_cmd = "" - return arcdir_set, [] + else: + raise ValueError("FATAL ERROR: Neither HPSSARCH nor LOCALARCH are set to True!") # Determine if we are archiving the EXPDIR this cycle (always skip for ensembles) if "enkf" not in arch_dict.RUN and arch_dict.ARCH_EXPDIR: @@ -123,7 +159,7 @@ def configure(self, arch_dict: Dict[str, Any]) -> (Dict[str, Any], List[Dict[str if arch_dict.ARCH_HASHES or arch_dict.ARCH_DIFFS: self._pop_git_info(arch_dict) - master_yaml = "master_" + arch_dict.RUN + ".yaml.j2" + master_yaml = "master_atardir_" + arch_dict.RUN + ".yaml.j2" parsed_sets = parse_j2yaml(os.path.join(archive_parm, master_yaml), arch_dict, @@ -138,7 +174,7 @@ def configure(self, arch_dict: Dict[str, Any]) -> (Dict[str, Any], List[Dict[str atardir_sets.append(dataset) - return arcdir_set, atardir_sets + return atardir_sets @logit(logger) def execute_store_products(self, arcdir_set: Dict[str, Any]) -> None: From a07e405d8a2d55860a4e74fe7f5351bc834b64c8 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Sat, 18 Jan 2025 19:38:34 -0500 Subject: [PATCH 027/105] modified archive.py --- ush/python/pygfs/task/archive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/python/pygfs/task/archive.py b/ush/python/pygfs/task/archive.py index 9835fa7654..36998d9d85 100644 --- a/ush/python/pygfs/task/archive.py +++ b/ush/python/pygfs/task/archive.py @@ -159,7 +159,7 @@ def configure_atardir(self, arch_dict: Dict[str, Any]) -> (List[Dict[str, Any]]) if arch_dict.ARCH_HASHES or arch_dict.ARCH_DIFFS: self._pop_git_info(arch_dict) - master_yaml = "master_atardir_" + arch_dict.RUN + ".yaml.j2" + master_yaml = "master_" + arch_dict.RUN + ".yaml.j2" parsed_sets = parse_j2yaml(os.path.join(archive_parm, master_yaml), arch_dict, From 95c0eb2f60b1edc83e84b281ec3db97e33a81e66 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Sat, 18 Jan 2025 19:53:26 -0500 Subject: [PATCH 028/105] modified configure function --- scripts/exglobal_archive_tars.py | 2 +- scripts/exglobal_archive_vrfy.py | 2 +- ush/python/pygfs/task/archive.py | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/exglobal_archive_tars.py b/scripts/exglobal_archive_tars.py index fb3f0952d8..e6ae4a41e9 100755 --- a/scripts/exglobal_archive_tars.py +++ b/scripts/exglobal_archive_tars.py @@ -57,7 +57,7 @@ def main(): with chdir(config.ROTDIR): # Determine which archives to create - atardir_sets = archive.configure_atardir(archive_dict) + atardir_sets = archive.configure_tars(archive_dict) # Create the backup tarballs and store in ATARDIR for atardir_set in atardir_sets: diff --git a/scripts/exglobal_archive_vrfy.py b/scripts/exglobal_archive_vrfy.py index a1a613151e..3315d72386 100755 --- a/scripts/exglobal_archive_vrfy.py +++ b/scripts/exglobal_archive_vrfy.py @@ -57,7 +57,7 @@ def main(): with chdir(config.ROTDIR): # Determine which archives to create - arcdir_set = archive.configure_arcdir(archive_dict) + arcdir_set = archive.configure_vrfy(archive_dict) # Populate the product archive (ARCDIR) archive.execute_store_products(arcdir_set) diff --git a/ush/python/pygfs/task/archive.py b/ush/python/pygfs/task/archive.py index 36998d9d85..5a5cb3a880 100644 --- a/ush/python/pygfs/task/archive.py +++ b/ush/python/pygfs/task/archive.py @@ -48,7 +48,7 @@ def __init__(self, config: Dict[str, Any]) -> None: self.archive_expdir = False @logit(logger) - def configure_arcdir(self, arch_dict: Dict[str, Any]) -> (Dict[str, Any]): + def configure_vrfy(self, arch_dict: Dict[str, Any]) -> (Dict[str, Any]): """Determine which files will need to be created to archive tp arcdir. Parameters @@ -87,14 +87,14 @@ def configure_arcdir(self, arch_dict: Dict[str, Any]) -> (Dict[str, Any]): arcdir_set = Archive._construct_arcdir_set(arcdir_j2yaml, arch_dict) - # Collect datasets that need to be archived + # Collect datasets that need to be archived self.tar_cmd = "" arcdir_set = (arcdir_set, []) return arcdir_set - + @logit(logger) - def configure_atardir(self, arch_dict: Dict[str, Any]) -> (List[Dict[str, Any]]): + def configure_tars(self, arch_dict: Dict[str, Any]) -> (List[Dict[str, Any]]): """Determine which tarballs will need to be created. Parameters From c0cdcb656288721525d2adfbe00a96922c6d46d3 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Sat, 18 Jan 2025 19:55:43 -0500 Subject: [PATCH 029/105] modified ush\python\pygfs\task\archive.py --- ush/python/pygfs/task/archive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/python/pygfs/task/archive.py b/ush/python/pygfs/task/archive.py index 5a5cb3a880..77b279d9a4 100644 --- a/ush/python/pygfs/task/archive.py +++ b/ush/python/pygfs/task/archive.py @@ -89,7 +89,7 @@ def configure_vrfy(self, arch_dict: Dict[str, Any]) -> (Dict[str, Any]): # Collect datasets that need to be archived self.tar_cmd = "" - arcdir_set = (arcdir_set, []) + arcdir_set = arcdir_set, [] return arcdir_set From da45368242d3f61a801ca3ec5cf40fa17f62b133 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Sat, 18 Jan 2025 19:58:40 -0500 Subject: [PATCH 030/105] modified ush\python\pygfs\task\archive.py --- ush/python/pygfs/task/archive.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ush/python/pygfs/task/archive.py b/ush/python/pygfs/task/archive.py index 77b279d9a4..4d5bac931f 100644 --- a/ush/python/pygfs/task/archive.py +++ b/ush/python/pygfs/task/archive.py @@ -89,7 +89,6 @@ def configure_vrfy(self, arch_dict: Dict[str, Any]) -> (Dict[str, Any]): # Collect datasets that need to be archived self.tar_cmd = "" - arcdir_set = arcdir_set, [] return arcdir_set From e7c3fc4b83f1970296c60f86535f8a5ee3819ca1 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Sat, 18 Jan 2025 20:06:54 -0500 Subject: [PATCH 031/105] modified ush\python\pygfs\task\archive.py --- ush/python/pygfs/task/archive.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ush/python/pygfs/task/archive.py b/ush/python/pygfs/task/archive.py index 4d5bac931f..731e6e0db4 100644 --- a/ush/python/pygfs/task/archive.py +++ b/ush/python/pygfs/task/archive.py @@ -89,6 +89,7 @@ def configure_vrfy(self, arch_dict: Dict[str, Any]) -> (Dict[str, Any]): # Collect datasets that need to be archived self.tar_cmd = "" + return arcdir_set, [] return arcdir_set From a0057611ccd75f5d07d0f62416d17a691116546a Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Sat, 18 Jan 2025 20:16:24 -0500 Subject: [PATCH 032/105] modified ush\python\pygfs\task\archive.py --- ush/python/pygfs/task/archive.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ush/python/pygfs/task/archive.py b/ush/python/pygfs/task/archive.py index 731e6e0db4..e23899aba2 100644 --- a/ush/python/pygfs/task/archive.py +++ b/ush/python/pygfs/task/archive.py @@ -89,9 +89,8 @@ def configure_vrfy(self, arch_dict: Dict[str, Any]) -> (Dict[str, Any]): # Collect datasets that need to be archived self.tar_cmd = "" - return arcdir_set, [] - return arcdir_set + return arcdir_set, [] @logit(logger) def configure_tars(self, arch_dict: Dict[str, Any]) -> (List[Dict[str, Any]]): From 5542a180d51ae12e9cf74d83e795a714d6696202 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Sat, 18 Jan 2025 20:19:27 -0500 Subject: [PATCH 033/105] modified ush\python\pygfs\task\archive.py --- ush/python/pygfs/task/archive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/python/pygfs/task/archive.py b/ush/python/pygfs/task/archive.py index e23899aba2..4d5bac931f 100644 --- a/ush/python/pygfs/task/archive.py +++ b/ush/python/pygfs/task/archive.py @@ -90,7 +90,7 @@ def configure_vrfy(self, arch_dict: Dict[str, Any]) -> (Dict[str, Any]): # Collect datasets that need to be archived self.tar_cmd = "" - return arcdir_set, [] + return arcdir_set @logit(logger) def configure_tars(self, arch_dict: Dict[str, Any]) -> (List[Dict[str, Any]]): From 7c1a012bb9b544607d3d453fdb0fd0585f3b62d6 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Sat, 18 Jan 2025 20:41:08 -0500 Subject: [PATCH 034/105] added inital changes to arch_vrfy and arch_tars of gfs job --- parm/config/gefs/config.resources | 10 +--------- parm/config/gfs/{config.arch => config.arch_tars} | 8 ++++---- parm/config/gfs/config.arch_vrfy | 15 +++++++++++++++ parm/config/gfs/config.base | 4 ++++ parm/config/gfs/config.resources | 2 +- workflow/applications/gfs_cycled.py | 10 ++++++++-- workflow/applications/gfs_forecast_only.py | 7 +++++-- 7 files changed, 38 insertions(+), 18 deletions(-) rename parm/config/gfs/{config.arch => config.arch_tars} (56%) create mode 100644 parm/config/gfs/config.arch_vrfy diff --git a/parm/config/gefs/config.resources b/parm/config/gefs/config.resources index 4102f3e4fe..7f24a080ac 100644 --- a/parm/config/gefs/config.resources +++ b/parm/config/gefs/config.resources @@ -323,15 +323,7 @@ case ${step} in export is_exclusive=False ;; - "arch_vrfy") - export walltime="06:00:00" - export ntasks=1 - export tasks_per_node=1 - export threads_per_task=1 - export memory="4096M" - ;; - - "arch_tars") + "arch_vrfy" | "arch_tars") export walltime="06:00:00" export ntasks=1 export tasks_per_node=1 diff --git a/parm/config/gfs/config.arch b/parm/config/gfs/config.arch_tars similarity index 56% rename from parm/config/gfs/config.arch rename to parm/config/gfs/config.arch_tars index a23bcce6ae..7605cc04c1 100644 --- a/parm/config/gfs/config.arch +++ b/parm/config/gfs/config.arch_tars @@ -1,15 +1,15 @@ #! /usr/bin/env bash -########## config.arch ########## +########## config.arch_tars ########## # Archive specific -echo "BEGIN: config.arch" +echo "BEGIN: config.arch_tars" # Get task specific resources -. "${EXPDIR}/config.resources" arch +. "${EXPDIR}/config.resources" arch_tars export ARCH_GAUSSIAN="YES" export ARCH_GAUSSIAN_FHMAX=${FHMAX_GFS} export ARCH_GAUSSIAN_FHINC=${FHOUT_GFS} -echo "END: config.arch" +echo "END: config.arch_tars" diff --git a/parm/config/gfs/config.arch_vrfy b/parm/config/gfs/config.arch_vrfy new file mode 100644 index 0000000000..cb668a48e2 --- /dev/null +++ b/parm/config/gfs/config.arch_vrfy @@ -0,0 +1,15 @@ +#! /usr/bin/env bash + +########## config.arch_vrfy ########## +# Archive specific + +echo "BEGIN: config.arch_vrfy" + +# Get task specific resources +. "${EXPDIR}/config.resources" arch_vrfy + +export ARCH_GAUSSIAN="YES" +export ARCH_GAUSSIAN_FHMAX=${FHMAX_GFS} +export ARCH_GAUSSIAN_FHINC=${FHOUT_GFS} + +echo "END: config.arch_vrfy" diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base index fbbe5c782f..8a79b9ec72 100644 --- a/parm/config/gfs/config.base +++ b/parm/config/gfs/config.base @@ -480,6 +480,10 @@ export LOCALARCH="@LOCALARCH@" # save data to local archive if [[ ${HPSSARCH} = "YES" ]] && [[ ${LOCALARCH} = "YES" ]]; then echo "FATAL ERROR: Both HPSS and local archiving selected. Please choose one or the other." exit 4 +elif [[ ${HPSSARCH} = "YES" ]] || [[ ${LOCALARCH} = "YES" ]]; then + export DO_ARCHTAR="YES" +else + export DO_ARCHTAR="NO" fi export ARCH_CYC=00 # Archive data at this cycle for warm start and/or forecast-only capabilities export ARCH_WARMICFREQ=4 # Archive frequency in days for warm start capability diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index aac06ff03c..26a5ca13aa 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -1054,7 +1054,7 @@ case ${step} in export is_exclusive=True ;; - "arch" | "earc" | "getic") + "arch_vrfy" | "arch_tars" | "earc" | "getic") walltime="06:00:00" ntasks=1 tasks_per_node=1 diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index 7a2c812a0d..89d35c4a5a 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -76,7 +76,7 @@ def _get_app_configs(self, run): if options['do_ocean'] or options['do_ice']: configs += ['oceanice_products'] - configs += ['stage_ic', 'sfcanl', 'analcalc', 'fcst', 'upp', 'atmos_products', 'arch', 'cleanup'] + configs += ['stage_ic', 'sfcanl', 'analcalc', 'fcst', 'upp', 'atmos_products', 'arch_vrfy', 'cleanup'] if options['do_hybvar']: if options['do_jediatmens']: @@ -146,6 +146,9 @@ def _get_app_configs(self, run): 'mos_stn_prdgen', 'mos_grd_prdgen', 'mos_ext_stn_prdgen', 'mos_ext_grd_prdgen', 'mos_wx_prdgen', 'mos_wx_ext_prdgen'] + if options['do_archtar']: + configs += ['arch_tars'] + return configs @staticmethod @@ -294,8 +297,11 @@ def get_task_names(self): 'mos_stn_prdgen', 'mos_grd_prdgen', 'mos_ext_stn_prdgen', 'mos_ext_grd_prdgen', 'mos_wx_prdgen', 'mos_wx_ext_prdgen'] + if options['do_archtar']: + task_names[run] += ['arch_tars'] + # Last two items - task_names[run] += ['arch', 'cleanup'] + task_names[run] += ['arch_vrfy', 'cleanup'] # Ensemble tasks elif 'enkf' in run: diff --git a/workflow/applications/gfs_forecast_only.py b/workflow/applications/gfs_forecast_only.py index de1c8cef27..40bb4cbb3c 100644 --- a/workflow/applications/gfs_forecast_only.py +++ b/workflow/applications/gfs_forecast_only.py @@ -29,7 +29,7 @@ def _get_app_configs(self, run): """ options = self.run_options[run] - configs = ['stage_ic', 'fcst', 'arch', 'cleanup'] + configs = ['stage_ic', 'fcst', 'arch_vrfy', 'cleanup'] if options['do_atm']: @@ -162,6 +162,9 @@ def get_task_names(self): 'mos_stn_prdgen', 'mos_grd_prdgen', 'mos_ext_stn_prdgen', 'mos_ext_grd_prdgen', 'mos_wx_prdgen', 'mos_wx_ext_prdgen'] - tasks += ['arch', 'cleanup'] # arch and cleanup **must** be the last tasks + if options['do_archtar']: + tasks += ['arch_tars'] + + tasks += ['arch_vrfy', 'cleanup'] # arch_tar, arch_vrfy, and cleanup **must** be the last tasks return {f"{self.run}": tasks} From b48d39d7f11f55a802b78c4dde96dce39fd00de7 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Sun, 19 Jan 2025 00:30:42 -0500 Subject: [PATCH 035/105] modified gefs_tasks.py --- workflow/rocoto/gefs_tasks.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index de57c5728c..83bfb9bfde 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -631,7 +631,9 @@ def arch_tars(self): def cleanup(self): deps = [] - dep_dict = {'type': 'task', 'name': 'gefs_arch'} + dep_dict = {'type': 'task', 'name': 'gefs_arch_vrfy'} + deps.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'task', 'name': 'gefs_arch_tars'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('cleanup') From be5e8e02e14dc7e329e7635513ca9471fff3b66d Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Sun, 19 Jan 2025 00:46:23 -0500 Subject: [PATCH 036/105] modified workflow\rocoto\gfs_tasks.py --- workflow/rocoto/gfs_tasks.py | 93 ++++++++++++++++++++++++++++++++++-- 1 file changed, 88 insertions(+), 5 deletions(-) diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 768512ba39..9e01f19873 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -2271,7 +2271,7 @@ def mos_wx_ext_prdgen(self): return task - def arch(self): + def arch_vrfy(self): deps = [] if self.app_config.mode in ['cycled']: if self.run in ['gfs']: @@ -2335,14 +2335,95 @@ def arch(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - resources = self.get_resource('arch') - task_name = f'{self.run}_arch' + resources = self.get_resource('arch_vrfy') + task_name = f'{self.run}_arch_vrfy' task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/arch.sh', + 'command': f'{self.HOMEgfs}/jobs/rocoto/arch_vrfy.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' + } + + task = rocoto.create_task(task_dict) + + return task + + def arch_tars(self): + deps = [] + if self.app_config.mode in ['cycled']: + if self.run in ['gfs']: + dep_dict = {'type': 'task', 'name': f'{self.run}_atmanlprod'} + deps.append(rocoto.add_dependency(dep_dict)) + if self.options['do_vminmon']: + dep_dict = {'type': 'task', 'name': f'{self.run}_vminmon'} + deps.append(rocoto.add_dependency(dep_dict)) + elif self.run in ['gdas']: + dep_dict = {'type': 'task', 'name': f'{self.run}_atmanlprod'} + deps.append(rocoto.add_dependency(dep_dict)) + if self.options['do_fit2obs']: + dep_dict = {'type': 'task', 'name': f'{self.run}_fit2obs'} + deps.append(rocoto.add_dependency(dep_dict)) + if self.options['do_verfozn']: + dep_dict = {'type': 'task', 'name': f'{self.run}_verfozn'} + deps.append(rocoto.add_dependency(dep_dict)) + if self.options['do_verfrad']: + dep_dict = {'type': 'task', 'name': f'{self.run}_verfrad'} + deps.append(rocoto.add_dependency(dep_dict)) + if self.options['do_vminmon']: + dep_dict = {'type': 'task', 'name': f'{self.run}_vminmon'} + deps.append(rocoto.add_dependency(dep_dict)) + if self.run in ['gfs'] and self.options['do_tracker']: + dep_dict = {'type': 'task', 'name': f'{self.run}_tracker'} + deps.append(rocoto.add_dependency(dep_dict)) + if self.run in ['gfs'] and self.options['do_genesis']: + dep_dict = {'type': 'task', 'name': f'{self.run}_genesis'} + deps.append(rocoto.add_dependency(dep_dict)) + if self.run in ['gfs'] and self.options['do_genesis_fsu']: + dep_dict = {'type': 'task', 'name': f'{self.run}_genesis_fsu'} + deps.append(rocoto.add_dependency(dep_dict)) + # Post job dependencies + dep_dict = {'type': 'metatask', 'name': f'{self.run}_atmos_prod'} + deps.append(rocoto.add_dependency(dep_dict)) + if self.options['do_wave']: + dep_dict = {'type': 'metatask', 'name': f'{self.run}_wavepostsbs'} + deps.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'task', 'name': f'{self.run}_wavepostpnt'} + deps.append(rocoto.add_dependency(dep_dict)) + if self.options['do_wave_bnd']: + dep_dict = {'type': 'task', 'name': f'{self.run}_wavepostbndpnt'} + deps.append(rocoto.add_dependency(dep_dict)) + if self.options['do_ocean']: + if self.run in ['gfs']: + dep_dict = {'type': 'metatask', 'name': f'{self.run}_ocean_prod'} + deps.append(rocoto.add_dependency(dep_dict)) + if self.options['do_ice']: + if self.run in ['gfs']: + dep_dict = {'type': 'metatask', 'name': f'{self.run}_ice_prod'} + deps.append(rocoto.add_dependency(dep_dict)) + # MOS job dependencies + if self.run in ['gfs'] and self.options['do_mos']: + mos_jobs = ["stn_prep", "grd_prep", "ext_stn_prep", "ext_grd_prep", + "stn_fcst", "grd_fcst", "ext_stn_fcst", "ext_grd_fcst", + "stn_prdgen", "grd_prdgen", "ext_stn_prdgen", "ext_grd_prdgen", + "wx_prdgen", "wx_ext_prdgen"] + for job in mos_jobs: + dep_dict = {'type': 'task', 'name': f'{self.run}_mos_{job}'} + deps.append(rocoto.add_dependency(dep_dict)) + + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + + resources = self.get_resource('arch_tars') + task_name = f'{self.run}_arch_tars' + task_dict = {'task_name': task_name, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars, + 'cycledef': self.run.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/arch_tars.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -2359,7 +2440,9 @@ def cleanup(self): dep_dict = {'type': 'metatask', 'name': f'{self.run}_eamn'} deps.append(rocoto.add_dependency(dep_dict)) else: - dep_dict = {'type': 'task', 'name': f'{self.run}_arch'} + dep_dict = {'type': 'task', 'name': f'{self.run}_arch_vrfy'} + deps.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'task', 'name': f'{self.run}_arch_tars'} deps.append(rocoto.add_dependency(dep_dict)) if self.options['do_gempak']: From e70fd60f177b202986dc7233b6b4312f769d421a Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Sun, 19 Jan 2025 16:43:58 -0500 Subject: [PATCH 037/105] modified tasks pys --- workflow/rocoto/gefs_tasks.py | 5 +++-- workflow/rocoto/gfs_tasks.py | 9 ++++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index 83bfb9bfde..8c6349e972 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -631,10 +631,11 @@ def arch_tars(self): def cleanup(self): deps = [] + if self.options['do_arch_tars']: + dep_dict = {'type': 'task', 'name': 'gefs_arch_tars'} + deps.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type': 'task', 'name': 'gefs_arch_vrfy'} deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'task', 'name': 'gefs_arch_tars'} - deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('cleanup') task_name = 'gefs_cleanup' diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 9e01f19873..e997908ec6 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -1867,13 +1867,16 @@ def fit2obs(self): def metp(self): deps = [] - dep_dict = {'type': 'task', 'name': f'{self.run}_arch'} + if self.options['do_arch_tars']: + dep_dict = {'type': 'task', 'name': 'gefs_arch_tars'} + deps.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'task', 'name': 'gefs_arch_vrfy'} deps.append(rocoto.add_dependency(dep_dict)) if self._base["interval_gfs"] < to_timedelta("24H"): n_lookback = self._base["interval_gfs"] // to_timedelta("6H") for lookback in range(1, n_lookback + 1): deps2 = [] - dep_dict = {'type': 'taskvalid', 'name': f'{self.run}_arch', 'condition': 'not'} + dep_dict = {'type': 'taskvalid', 'name': f'{self.run}_arch_vrfy', 'condition': 'not'} deps2.append(rocoto.add_dependency(dep_dict)) for lookback2 in range(1, lookback): offset = timedelta_to_HMS(-to_timedelta(f'{6*lookback2}H')) @@ -1881,7 +1884,7 @@ def metp(self): deps2.append(rocoto.add_dependency(dep_dict)) offset = timedelta_to_HMS(-to_timedelta(f'{6*lookback}H')) - dep_dict = {'type': 'task', 'name': f'{self.run}_arch', 'offset': offset} + dep_dict = {'type': 'task', 'name': f'{self.run}_arch_vrfy', 'offset': offset} deps2.append(rocoto.add_dependency(dep_dict)) deps.append(rocoto.create_dependency(dep_condition='and', dep=deps2)) From 6049f9197f21b28c06b7691f8c47952164c74626 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Sun, 19 Jan 2025 16:50:31 -0500 Subject: [PATCH 038/105] modified gfs_tasks --- workflow/rocoto/gefs_tasks.py | 2 +- workflow/rocoto/gfs_tasks.py | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index 8c6349e972..0b6e3d1b60 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -631,7 +631,7 @@ def arch_tars(self): def cleanup(self): deps = [] - if self.options['do_arch_tars']: + if self.options['do_archtar']: dep_dict = {'type': 'task', 'name': 'gefs_arch_tars'} deps.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type': 'task', 'name': 'gefs_arch_vrfy'} diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index e997908ec6..09b48a299d 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -1867,10 +1867,10 @@ def fit2obs(self): def metp(self): deps = [] - if self.options['do_arch_tars']: - dep_dict = {'type': 'task', 'name': 'gefs_arch_tars'} + if self.options['do_archtar']: + dep_dict = {'type': 'task', 'name': f'{self.run}_arch_tars'} deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'task', 'name': 'gefs_arch_vrfy'} + dep_dict = {'type': 'task', 'name': f'{self.run}_arch_vrfy'} deps.append(rocoto.add_dependency(dep_dict)) if self._base["interval_gfs"] < to_timedelta("24H"): n_lookback = self._base["interval_gfs"] // to_timedelta("6H") @@ -2445,8 +2445,9 @@ def cleanup(self): else: dep_dict = {'type': 'task', 'name': f'{self.run}_arch_vrfy'} deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'task', 'name': f'{self.run}_arch_tars'} - deps.append(rocoto.add_dependency(dep_dict)) + if self.options['do_archtar']: + dep_dict = {'type': 'task', 'name': f'{self.run}_arch_tars'} + deps.append(rocoto.add_dependency(dep_dict)) if self.options['do_gempak']: if self.run in ['gdas']: From 2dc146350d0354e65cf27db4854bf5628226f079 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Sun, 19 Jan 2025 17:23:25 -0500 Subject: [PATCH 039/105] modified arch_tars var --- parm/config/gefs/config.resources.AWSPW | 2 +- parm/config/gfs/config.resources.AWSPW | 2 +- parm/config/gfs/config.resources.WCOSS2 | 2 +- workflow/applications/gfs_forecast_only.py | 3 +++ 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/parm/config/gefs/config.resources.AWSPW b/parm/config/gefs/config.resources.AWSPW index f91460b6aa..43cfcf56cc 100644 --- a/parm/config/gefs/config.resources.AWSPW +++ b/parm/config/gefs/config.resources.AWSPW @@ -18,7 +18,7 @@ case ${step} in max_tasks_per_node=48 ;; - "arch") + "arch_vrfy" | "arch_tars") export PARTITION_BATCH="process" max_tasks_per_node=24 ;; diff --git a/parm/config/gfs/config.resources.AWSPW b/parm/config/gfs/config.resources.AWSPW index 22fe110670..d044c475fb 100644 --- a/parm/config/gfs/config.resources.AWSPW +++ b/parm/config/gfs/config.resources.AWSPW @@ -18,7 +18,7 @@ case ${step} in max_tasks_per_node=48 ;; - "arch") + "arch_vrfy" | "arch_tars") export PARTITION_BATCH="process" max_tasks_per_node=24 ;; diff --git a/parm/config/gfs/config.resources.WCOSS2 b/parm/config/gfs/config.resources.WCOSS2 index 342286d008..5aa6c3d15f 100644 --- a/parm/config/gfs/config.resources.WCOSS2 +++ b/parm/config/gfs/config.resources.WCOSS2 @@ -37,7 +37,7 @@ case ${step} in export memory="200GB" ;; - "arch" | "earc" | "getic") + "arch_vrfy" | "arch_tars" | "earc" | "getic") declare -x "memory"="50GB" ;; diff --git a/workflow/applications/gfs_forecast_only.py b/workflow/applications/gfs_forecast_only.py index 40bb4cbb3c..7d2673568e 100644 --- a/workflow/applications/gfs_forecast_only.py +++ b/workflow/applications/gfs_forecast_only.py @@ -81,6 +81,9 @@ def _get_app_configs(self, run): 'mos_stn_prdgen', 'mos_grd_prdgen', 'mos_ext_stn_prdgen', 'mos_ext_grd_prdgen', 'mos_wx_prdgen', 'mos_wx_ext_prdgen'] + if options['do_archtar']: + configs += ['arc_tars'] + return configs @staticmethod From ee8f6b1245155f4c61d68f6cca6f9e4b28c98f00 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Sun, 19 Jan 2025 17:25:28 -0500 Subject: [PATCH 040/105] corrected a typo --- workflow/applications/gfs_forecast_only.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/applications/gfs_forecast_only.py b/workflow/applications/gfs_forecast_only.py index 7d2673568e..b836505c31 100644 --- a/workflow/applications/gfs_forecast_only.py +++ b/workflow/applications/gfs_forecast_only.py @@ -82,7 +82,7 @@ def _get_app_configs(self, run): 'mos_wx_prdgen', 'mos_wx_ext_prdgen'] if options['do_archtar']: - configs += ['arc_tars'] + configs += ['arch_tars'] return configs From d4ae8fc141c361fcf81e9225293f727a32d462c6 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Mon, 20 Jan 2025 14:04:49 -0500 Subject: [PATCH 041/105] modified earc job --- ...S_ENKF_ARCHIVE => JGDAS_ENKF_ARCHIVE_VRFY} | 4 +- jobs/rocoto/JGDAS_ENKF_ARCHIVE_TARS | 44 ++++++++++++++ .../gfs/{config.earc => config.earc_tars} | 10 ++-- parm/config/gfs/config.earc_vrfy | 35 +++++++++++ ..._enkf_earc.py => exgdas_enkf_earc_tars.py} | 5 +- scripts/exgdas_enkf_earc_vrfy.py | 58 +++++++++++++++++++ workflow/rocoto/gfs_tasks.py | 50 +++++++++++++++- 7 files changed, 192 insertions(+), 14 deletions(-) rename jobs/{JGDAS_ENKF_ARCHIVE => JGDAS_ENKF_ARCHIVE_VRFY} (91%) mode change 100755 => 100644 create mode 100644 jobs/rocoto/JGDAS_ENKF_ARCHIVE_TARS rename parm/config/gfs/{config.earc => config.earc_tars} (73%) create mode 100644 parm/config/gfs/config.earc_vrfy rename scripts/{exgdas_enkf_earc.py => exgdas_enkf_earc_tars.py} (92%) mode change 100755 => 100644 create mode 100644 scripts/exgdas_enkf_earc_vrfy.py diff --git a/jobs/JGDAS_ENKF_ARCHIVE b/jobs/JGDAS_ENKF_ARCHIVE_VRFY old mode 100755 new mode 100644 similarity index 91% rename from jobs/JGDAS_ENKF_ARCHIVE rename to jobs/JGDAS_ENKF_ARCHIVE_VRFY index 021c454afc..7940edd744 --- a/jobs/JGDAS_ENKF_ARCHIVE +++ b/jobs/JGDAS_ENKF_ARCHIVE_VRFY @@ -1,7 +1,7 @@ #! /usr/bin/env bash source "${HOMEgfs}/ush/preamble.sh" -source "${HOMEgfs}/ush/jjob_header.sh" -e "earc" -c "base earc" +source "${HOMEgfs}/ush/jjob_header.sh" -e "earc_vrfy" -c "base earc_vrfy" ############################################## @@ -17,7 +17,7 @@ MEMDIR="ensstat" YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ # Run archive script ############################################################### -"${SCRgfs}/exgdas_enkf_earc.py" +"${SCRgfs}/exgdas_enkf_earc_vrfy.py" status=$? [[ ${status} -ne 0 ]] && exit "${status}" diff --git a/jobs/rocoto/JGDAS_ENKF_ARCHIVE_TARS b/jobs/rocoto/JGDAS_ENKF_ARCHIVE_TARS new file mode 100644 index 0000000000..04fe8e3141 --- /dev/null +++ b/jobs/rocoto/JGDAS_ENKF_ARCHIVE_TARS @@ -0,0 +1,44 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" +source "${HOMEgfs}/ush/jjob_header.sh" -e "earc_tars" -c "base earc_tars" + + +############################################## +# Set variables used in the script +############################################## +YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_TOP +MEMDIR="ensstat" YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ + COMIN_ATMOS_ANALYSIS_ENSSTAT:COM_ATMOS_ANALYSIS_TMPL \ + COMIN_ATMOS_HISTORY_ENSSTAT:COM_ATMOS_HISTORY_TMPL \ + COMIN_SNOW_ANALYSIS_ENSSTAT:COM_SNOW_ANALYSIS_TMPL + +############################################################### +# Run archive script +############################################################### + +"${SCRgfs}/exgdas_enkf_earc_tars.py" +status=$? +[[ ${status} -ne 0 ]] && exit "${status}" + +############################################################### + +############################################## +# End JOB SPECIFIC work +############################################## + +############################################## +# Final processing +############################################## +if [[ -e "${pgmout}" ]] ; then + cat "${pgmout}" +fi + + +########################################## +# Remove the Temporary working directory +########################################## +cd "${DATAROOT}" || (echo "${DATAROOT} does not exist. ABORT!"; exit 1) +[[ ${KEEPDATA} = "NO" ]] && rm -rf "${DATA}" + +exit 0 diff --git a/parm/config/gfs/config.earc b/parm/config/gfs/config.earc_tars similarity index 73% rename from parm/config/gfs/config.earc rename to parm/config/gfs/config.earc_tars index 00a2fa95bd..b4a5856118 100644 --- a/parm/config/gfs/config.earc +++ b/parm/config/gfs/config.earc_tars @@ -1,14 +1,14 @@ #! /usr/bin/env bash -########## config.earc ########## +########## config.earc_tars ########## # Ensemble archive specific -echo "BEGIN: config.earc" +echo "BEGIN: config.earc_tars" # Get task specific resources -. $EXPDIR/config.resources earc +. $EXPDIR/config.resources earc_tars -# Set the number of ensemble members to archive per earc job +# Set the number of ensemble members to archive per earc_tars job case "${CASE_ENS}" in "C48" | "C96") export NMEM_EARCGRP=80 @@ -32,4 +32,4 @@ esac export RMOLDSTD_ENKF=144 export RMOLDEND_ENKF=24 -echo "END: config.earc" +echo "END: config.earc_tars" diff --git a/parm/config/gfs/config.earc_vrfy b/parm/config/gfs/config.earc_vrfy new file mode 100644 index 0000000000..9d1f0cfc19 --- /dev/null +++ b/parm/config/gfs/config.earc_vrfy @@ -0,0 +1,35 @@ +#! /usr/bin/env bash + +########## config.earc_vrfy ########## +# Ensemble archive specific + +echo "BEGIN: config.earc_vrfy" + +# Get task specific resources +. $EXPDIR/config.resources earc_vrfy + +# Set the number of ensemble members to archive per earc_vrfy job +case "${CASE_ENS}" in + "C48" | "C96") + export NMEM_EARCGRP=80 + ;; + "C192") + export NMEM_EARCGRP=20 + ;; + "C384" | "C768") + export NMEM_EARCGRP=10 + ;; + "C1152") + export NMEM_EARCGRP=4 + ;; + *) + echo "FATAL ERROR: Unknown ensemble resolution ${CASE_ENS}, ABORT!" + exit 1 + ;; +esac + +#--starting and ending hours of previous cycles to be removed from rotating directory +export RMOLDSTD_ENKF=144 +export RMOLDEND_ENKF=24 + +echo "END: config.earc_vrfy" diff --git a/scripts/exgdas_enkf_earc.py b/scripts/exgdas_enkf_earc_tars.py old mode 100755 new mode 100644 similarity index 92% rename from scripts/exgdas_enkf_earc.py rename to scripts/exgdas_enkf_earc_tars.py index 107d541a41..63c978d6a8 --- a/scripts/exgdas_enkf_earc.py +++ b/scripts/exgdas_enkf_earc_tars.py @@ -46,10 +46,7 @@ def main(): os.chdir(config.ROTDIR) # Determine which archives to create - arcdir_set, atardir_sets = archive.configure(archive_dict) - - # Populate the product archive (ARCDIR) - archive.execute_store_products(arcdir_set) + atardir_sets = archive.configure_tars(archive_dict) # Create the backup tarballs and store in ATARDIR for atardir_set in atardir_sets: diff --git a/scripts/exgdas_enkf_earc_vrfy.py b/scripts/exgdas_enkf_earc_vrfy.py new file mode 100644 index 0000000000..da893c13ae --- /dev/null +++ b/scripts/exgdas_enkf_earc_vrfy.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 + +import os + +from pygfs.task.archive import Archive +from wxflow import AttrDict, Logger, cast_strdict_as_dtypedict, chdir, logit + +# initialize root logger +logger = Logger(level=os.environ.get("LOGGING_LEVEL", "DEBUG"), colored_log=True) + + +@logit(logger) +def main(): + + config = cast_strdict_as_dtypedict(os.environ) + + # Instantiate the Archive object + archive = Archive(config) + + # Pull out all the configuration keys needed to run the rest of archive steps + keys = ['ATARDIR', 'current_cycle', 'IAUFHRS', 'RUN', 'PDY', + 'PSLOT', 'HPSSARCH', 'LOCALARCH', 'ROTDIR', 'PARMgfs', + 'ARCDIR', 'SDATE', 'MODE', 'ENSGRP', 'NMEM_EARCGRP', + 'NMEM_ENS', 'DO_CALC_INCREMENT_ENKF_GFS', 'DO_JEDIATMENS', + 'lobsdiag_forenkf', 'FHMIN_ENKF', 'FHMAX_ENKF_GFS', + 'FHOUT_ENKF_GFS', 'FHMAX_ENKF', 'FHOUT_ENKF', 'ENKF_SPREAD', + 'restart_interval_enkfgdas', 'restart_interval_enkfgfs', + 'DOHYBVAR', 'DOIAU_ENKF', 'IAU_OFFSET', 'DOIAU', 'DO_CA', + 'DO_CALC_INCREMENT', 'assim_freq', 'ARCH_CYC', 'DO_JEDISNOWDA', + 'ARCH_WARMICFREQ', 'ARCH_FCSTICFREQ', + 'IAUFHRS_ENKF', 'NET', 'NMEM_ENS_GFS'] + + archive_dict = AttrDict() + for key in keys: + archive_dict[key] = archive.task_config.get(key) + if archive_dict[key] is None: + print(f"Warning: key ({key}) not found in task_config!") + + # Also import all COMIN* directory and template variables + for key in archive.task_config.keys(): + if key.startswith("COM"): + archive_dict[key] = archive.task_config[key] + + cwd = os.getcwd() + + os.chdir(config.ROTDIR) + + # Determine which archives to create + arcdir_set = archive.configure_vrfy(archive_dict) + + # Populate the product archive (ARCDIR) + archive.execute_store_products(arcdir_set) + + os.chdir(cwd) + + +if __name__ == '__main__': + main() diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 09b48a299d..8c620b6450 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -3001,7 +3001,7 @@ def _get_eposgroups(epos): return task - def earc(self): + def earc_vrfy(self): deps = [] if 'enkfgdas' in self.run: @@ -3020,7 +3020,7 @@ def earc(self): n_groups = -(self.nmem // -self._configs['earc']['NMEM_EARCGRP']) groups = ' '.join([f'{grp:02d}' for grp in range(0, n_groups + 1)]) - resources = self.get_resource('earc') + resources = self.get_resource('earc_tars') var_dict = {'grp': groups} @@ -3030,7 +3030,7 @@ def earc(self): 'dependency': dependencies, 'envars': earcenvars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/earc.sh', + 'command': f'{self.HOMEgfs}/jobs/rocoto/earc_vrfy.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -3044,3 +3044,47 @@ def earc(self): task = rocoto.create_task(metatask_dict) return task + + def earc_tars(self): + + deps = [] + if 'enkfgdas' in self.run: + dep_dict = {'type': 'metatask', 'name': f'{self.run}_epmn'} + else: + dep_dict = {'type': 'task', 'name': f'{self.run}_esfc'} + deps.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'task', 'name': f'{self.run}_echgres'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + + earcenvars = self.envars.copy() + earcenvars.append(rocoto.create_envar(name='ENSGRP', value='#grp#')) + + # Integer division is floor division, but we need ceiling division + n_groups = -(self.nmem // -self._configs['earc']['NMEM_EARCGRP']) + groups = ' '.join([f'{grp:02d}' for grp in range(0, n_groups + 1)]) + + resources = self.get_resource('earc_tars') + + var_dict = {'grp': groups} + + task_name = f'{self.run}_earc#grp#' + task_dict = {'task_name': task_name, + 'resources': resources, + 'dependency': dependencies, + 'envars': earcenvars, + 'cycledef': self.run.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/earc_tars.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' + } + + metatask_dict = {'task_name': f'{self.run}_eamn', + 'var_dict': var_dict, + 'task_dict': task_dict + } + + task = rocoto.create_task(metatask_dict) + + return task \ No newline at end of file From 1dfcabaf67748e114d75da335b60b79b19551f2c Mon Sep 17 00:00:00 2001 From: Anton Fernando Date: Mon, 20 Jan 2025 19:31:04 +0000 Subject: [PATCH 042/105] modified permssion --- jobs/JGDAS_ENKF_ARCHIVE_VRFY | 0 parm/config/gfs/config.earc_tars | 0 parm/config/gfs/config.earc_vrfy | 0 scripts/exgdas_enkf_earc_tars.py | 0 scripts/exgdas_enkf_earc_vrfy.py | 0 5 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 jobs/JGDAS_ENKF_ARCHIVE_VRFY mode change 100644 => 100755 parm/config/gfs/config.earc_tars mode change 100644 => 100755 parm/config/gfs/config.earc_vrfy mode change 100644 => 100755 scripts/exgdas_enkf_earc_tars.py mode change 100644 => 100755 scripts/exgdas_enkf_earc_vrfy.py diff --git a/jobs/JGDAS_ENKF_ARCHIVE_VRFY b/jobs/JGDAS_ENKF_ARCHIVE_VRFY old mode 100644 new mode 100755 diff --git a/parm/config/gfs/config.earc_tars b/parm/config/gfs/config.earc_tars old mode 100644 new mode 100755 diff --git a/parm/config/gfs/config.earc_vrfy b/parm/config/gfs/config.earc_vrfy old mode 100644 new mode 100755 diff --git a/scripts/exgdas_enkf_earc_tars.py b/scripts/exgdas_enkf_earc_tars.py old mode 100644 new mode 100755 diff --git a/scripts/exgdas_enkf_earc_vrfy.py b/scripts/exgdas_enkf_earc_vrfy.py old mode 100644 new mode 100755 From c57bf1209e66b2c0cac2845f1b31b2b841ab0816 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Mon, 20 Jan 2025 14:40:17 -0500 Subject: [PATCH 043/105] modified jobs\JGDAS_ENKF_ARCHIVE_VRFY --- jobs/JGDAS_ENKF_ARCHIVE_VRFY | 1 - 1 file changed, 1 deletion(-) diff --git a/jobs/JGDAS_ENKF_ARCHIVE_VRFY b/jobs/JGDAS_ENKF_ARCHIVE_VRFY index 7940edd744..0094401987 100755 --- a/jobs/JGDAS_ENKF_ARCHIVE_VRFY +++ b/jobs/JGDAS_ENKF_ARCHIVE_VRFY @@ -34,7 +34,6 @@ if [[ -e "${pgmout}" ]] ; then cat "${pgmout}" fi - ########################################## # Remove the Temporary working directory ########################################## From 1822c177aaa3c218b64d5de7fefcddabd5caf9ef Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Mon, 20 Jan 2025 14:44:34 -0500 Subject: [PATCH 044/105] added a blank line to workflow\rocoto\gfs_tasks.py --- workflow/rocoto/gfs_tasks.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 8c620b6450..969267f8d4 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -3087,4 +3087,5 @@ def earc_tars(self): task = rocoto.create_task(metatask_dict) - return task \ No newline at end of file + return task + \ No newline at end of file From 4ee399c26481eb32527196afc0de2c5002ba4d1b Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Mon, 20 Jan 2025 14:45:30 -0500 Subject: [PATCH 045/105] modified workflow\rocoto\gfs_tasks.py --- workflow/rocoto/gfs_tasks.py | 1 - 1 file changed, 1 deletion(-) diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 969267f8d4..af00486cd2 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -3088,4 +3088,3 @@ def earc_tars(self): task = rocoto.create_task(metatask_dict) return task - \ No newline at end of file From 2de386615e8470e2dc82a1b7f80230a5be2a6ca0 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Mon, 20 Jan 2025 14:53:05 -0500 Subject: [PATCH 046/105] modified earc jobs --- jobs/rocoto/{earc.sh => earc_tars.sh} | 4 ++-- jobs/rocoto/earc_vrfy.sh | 24 ++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) rename jobs/rocoto/{earc.sh => earc_tars.sh} (90%) mode change 100755 => 100644 create mode 100644 jobs/rocoto/earc_vrfy.sh diff --git a/jobs/rocoto/earc.sh b/jobs/rocoto/earc_tars.sh old mode 100755 new mode 100644 similarity index 90% rename from jobs/rocoto/earc.sh rename to jobs/rocoto/earc_tars.sh index 4a9263b509..a796d90232 --- a/jobs/rocoto/earc.sh +++ b/jobs/rocoto/earc_tars.sh @@ -13,12 +13,12 @@ status=$? PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${HOMEgfs}/ush/python" export PYTHONPATH -export job="earc" +export job="earc_tars" export jobid="${job}.$$" ############################################################### # Execute the JJOB -"${HOMEgfs}/jobs/JGDAS_ENKF_ARCHIVE" +"${HOMEgfs}/jobs/JGDAS_ENKF_ARCHIVE_TARS" status=$? exit "${status}" diff --git a/jobs/rocoto/earc_vrfy.sh b/jobs/rocoto/earc_vrfy.sh new file mode 100644 index 0000000000..2b8c0db05b --- /dev/null +++ b/jobs/rocoto/earc_vrfy.sh @@ -0,0 +1,24 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" + +############################################################### +# Source FV3GFS workflow modules +. "${HOMEgfs}/ush/load_fv3gfs_modules.sh" +status=$? +[[ ${status} -ne 0 ]] && exit "${status}" + +############################################################### +# setup python path for workflow utilities and tasks +PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${HOMEgfs}/ush/python" +export PYTHONPATH + +export job="earc_vrfy" +export jobid="${job}.$$" + +############################################################### +# Execute the JJOB +"${HOMEgfs}/jobs/JGDAS_ENKF_ARCHIVE_VRFY" +status=$? + +exit "${status}" From 43055d6fb5bcc12bb1306b5167ff2e3a09bfb87f Mon Sep 17 00:00:00 2001 From: Anton Fernando Date: Mon, 20 Jan 2025 19:55:19 +0000 Subject: [PATCH 047/105] changed permission to files --- jobs/rocoto/earc_tars.sh | 0 jobs/rocoto/earc_vrfy.sh | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 jobs/rocoto/earc_tars.sh mode change 100644 => 100755 jobs/rocoto/earc_vrfy.sh diff --git a/jobs/rocoto/earc_tars.sh b/jobs/rocoto/earc_tars.sh old mode 100644 new mode 100755 diff --git a/jobs/rocoto/earc_vrfy.sh b/jobs/rocoto/earc_vrfy.sh old mode 100644 new mode 100755 From a57935808732a61dc4f5d2f8becad7b52c0053c5 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Mon, 20 Jan 2025 15:07:14 -0500 Subject: [PATCH 048/105] changed earc var --- .github/CODEOWNERS | 3 ++- parm/config/gefs/config.base | 2 +- parm/config/gfs/config.resources | 2 +- parm/config/gfs/config.resources.WCOSS2 | 2 +- workflow/applications/gfs_cycled.py | 4 ++-- workflow/rocoto/gfs_tasks.py | 8 ++++---- workflow/rocoto/tasks.py | 4 ++-- 7 files changed, 13 insertions(+), 12 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index b0b51922c5..81e387d361 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -76,7 +76,8 @@ scripts/exgdas_atmos_gempak_gif_ncdc.sh @GwenChen-NOAA scripts/exgdas_atmos_nawips.sh @GwenChen-NOAA scripts/exgdas_atmos_verfozn.sh @EdwardSafford-NOAA scripts/exgdas_atmos_verfrad.sh @EdwardSafford-NOAA -scripts/exgdas_enkf_earc.py @DavidHuber-NOAA +scripts/exgdas_enkf_earc_vrfy.py @DavidHuber-NOAA +scripts/exgdas_enkf_earc_tars.py @DavidHuber-NOAA scripts/exgdas_enkf_ecen.sh @CoryMartin-NOAA @RussTreadon-NOAA @CatherineThomas-NOAA scripts/exgdas_enkf_post.sh @CoryMartin-NOAA @RussTreadon-NOAA @CatherineThomas-NOAA scripts/exgdas_enkf_select_obs.sh @CoryMartin-NOAA @RussTreadon-NOAA @CatherineThomas-NOAA diff --git a/parm/config/gefs/config.base b/parm/config/gefs/config.base index db6e4bdaed..cbcba0cde7 100644 --- a/parm/config/gefs/config.base +++ b/parm/config/gefs/config.base @@ -352,7 +352,7 @@ export ARCH_EXPDIR_FREQ=0 # How often to archive the EXPDIR in hours or 0 for export ARCH_HASHES='YES' # Archive the hashes of the GW and submodules and 'git status' for each; requires ARCH_EXPDIR export ARCH_DIFFS='NO' # Archive the output of 'git diff' for the GW; requires ARCH_EXPDIR -export DELETE_COM_IN_ARCHIVE_JOB="YES" # NO=retain ROTDIR. YES default in arch.sh and earc.sh. +export DELETE_COM_IN_ARCHIVE_JOB="YES" # NO=retain ROTDIR. YES default in arch_*.sh and earc_*.sh. # Number of regional collectives to create soundings for export NUM_SND_COLLECTIVES=${NUM_SND_COLLECTIVES:-9} diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index 26a5ca13aa..35ef6826c9 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -1054,7 +1054,7 @@ case ${step} in export is_exclusive=True ;; - "arch_vrfy" | "arch_tars" | "earc" | "getic") + "arch_vrfy" | "arch_tars" | "earc_vrfy" | "earc_tars" | "getic") walltime="06:00:00" ntasks=1 tasks_per_node=1 diff --git a/parm/config/gfs/config.resources.WCOSS2 b/parm/config/gfs/config.resources.WCOSS2 index 5aa6c3d15f..be55214cac 100644 --- a/parm/config/gfs/config.resources.WCOSS2 +++ b/parm/config/gfs/config.resources.WCOSS2 @@ -37,7 +37,7 @@ case ${step} in export memory="200GB" ;; - "arch_vrfy" | "arch_tars" | "earc" | "getic") + "arch_vrfy" | "arch_tars" | "earc_vrfy" | "earc_tars" | "getic") declare -x "memory"="50GB" ;; diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index 89d35c4a5a..2b4c934a16 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -84,7 +84,7 @@ def _get_app_configs(self, run): 'atmensanlletkf', 'atmensanlfv3inc', 'atmensanlfinal'] else: configs += ['eobs', 'eomg', 'ediag', 'eupd'] - configs += ['ecen', 'esfc', 'efcs', 'echgres', 'epos', 'earc'] + configs += ['ecen', 'esfc', 'efcs', 'echgres', 'epos', 'earc_vrfy'] if options['do_fit2obs']: configs += ['fit2obs'] @@ -323,6 +323,6 @@ def get_task_names(self): task_names[run].append('efcs') if 'gdas' in run else 0 task_names[run].append('epos') if 'gdas' in run else 0 - task_names[run] += ['stage_ic', 'ecen', 'esfc', 'earc', 'cleanup'] + task_names[run] += ['stage_ic', 'ecen', 'esfc', 'earc_vrfy', 'cleanup'] return task_names diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index af00486cd2..7051038b18 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -3017,14 +3017,14 @@ def earc_vrfy(self): earcenvars.append(rocoto.create_envar(name='ENSGRP', value='#grp#')) # Integer division is floor division, but we need ceiling division - n_groups = -(self.nmem // -self._configs['earc']['NMEM_EARCGRP']) + n_groups = -(self.nmem // -self._configs['earc_vrfy']['NMEM_EARCGRP']) groups = ' '.join([f'{grp:02d}' for grp in range(0, n_groups + 1)]) resources = self.get_resource('earc_tars') var_dict = {'grp': groups} - task_name = f'{self.run}_earc#grp#' + task_name = f'{self.run}_earc_vrfy_#grp#' task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, @@ -3061,14 +3061,14 @@ def earc_tars(self): earcenvars.append(rocoto.create_envar(name='ENSGRP', value='#grp#')) # Integer division is floor division, but we need ceiling division - n_groups = -(self.nmem // -self._configs['earc']['NMEM_EARCGRP']) + n_groups = -(self.nmem // -self._configs['earc_tars']['NMEM_EARCGRP']) groups = ' '.join([f'{grp:02d}' for grp in range(0, n_groups + 1)]) resources = self.get_resource('earc_tars') var_dict = {'grp': groups} - task_name = f'{self.run}_earc#grp#' + task_name = f'{self.run}_earc_tars_#grp#' task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, diff --git a/workflow/rocoto/tasks.py b/workflow/rocoto/tasks.py index d82e875753..aa11e67f34 100644 --- a/workflow/rocoto/tasks.py +++ b/workflow/rocoto/tasks.py @@ -11,13 +11,13 @@ class Tasks: - SERVICE_TASKS = ['arch_vrfy', 'arch_tars', 'earc', 'stage_ic', 'cleanup'] + SERVICE_TASKS = ['arch_vrfy', 'arch_tars', 'earc_vrfy', 'earc_tars', 'stage_ic', 'cleanup'] VALID_TASKS = ['aerosol_init', 'stage_ic', 'prep', 'anal', 'sfcanl', 'analcalc', 'analdiag', 'arch_vrfy', 'arch_tars', "cleanup", 'prepatmiodaobs', 'atmanlinit', 'atmanlvar', 'atmanlfv3inc', 'atmanlfinal', 'prepoceanobs', 'marineanlinit', 'marineanlletkf', 'marinebmat', 'marineanlvar', 'ocnanalecen', 'marineanlchkpt', 'marineanlfinal', 'ocnanalvrfy', - 'earc', 'ecen', 'echgres', 'ediag', 'efcs', + 'earc_vrfy', 'earc_tars', 'ecen', 'echgres', 'ediag', 'efcs', 'eobs', 'eomg', 'epos', 'esfc', 'eupd', 'atmensanlinit', 'atmensanlobs', 'atmensanlsol', 'atmensanlletkf', 'atmensanlfv3inc', 'atmensanlfinal', 'aeroanlinit', 'aeroanlvar', 'aeroanlfinal', 'aeroanlgenb', From 8fa12b88020e74f2b886784522449434dbe57048 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Mon, 20 Jan 2025 15:13:20 -0500 Subject: [PATCH 049/105] modified workflow\applications\gfs_cycled.py --- workflow/applications/gfs_cycled.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index 2b4c934a16..5b0581dd9b 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -84,7 +84,7 @@ def _get_app_configs(self, run): 'atmensanlletkf', 'atmensanlfv3inc', 'atmensanlfinal'] else: configs += ['eobs', 'eomg', 'ediag', 'eupd'] - configs += ['ecen', 'esfc', 'efcs', 'echgres', 'epos', 'earc_vrfy'] + configs += ['ecen', 'esfc', 'efcs', 'echgres', 'epos', 'earc_vrfy', 'earc_vrfy'] if options['do_fit2obs']: configs += ['fit2obs'] @@ -323,6 +323,6 @@ def get_task_names(self): task_names[run].append('efcs') if 'gdas' in run else 0 task_names[run].append('epos') if 'gdas' in run else 0 - task_names[run] += ['stage_ic', 'ecen', 'esfc', 'earc_vrfy', 'cleanup'] + task_names[run] += ['stage_ic', 'ecen', 'esfc', 'earc_vrfy', 'earc_vrfy', 'cleanup'] return task_names From c5b95c8e24aee5bd86beaec0f8f280c0a5e61605 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Mon, 20 Jan 2025 15:13:54 -0500 Subject: [PATCH 050/105] modified workflow\applications\gfs_cycled.py --- workflow/applications/gfs_cycled.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index 5b0581dd9b..3fdb812659 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -84,7 +84,7 @@ def _get_app_configs(self, run): 'atmensanlletkf', 'atmensanlfv3inc', 'atmensanlfinal'] else: configs += ['eobs', 'eomg', 'ediag', 'eupd'] - configs += ['ecen', 'esfc', 'efcs', 'echgres', 'epos', 'earc_vrfy', 'earc_vrfy'] + configs += ['ecen', 'esfc', 'efcs', 'echgres', 'epos', 'earc_vrfy', 'earc_tars'] if options['do_fit2obs']: configs += ['fit2obs'] @@ -323,6 +323,6 @@ def get_task_names(self): task_names[run].append('efcs') if 'gdas' in run else 0 task_names[run].append('epos') if 'gdas' in run else 0 - task_names[run] += ['stage_ic', 'ecen', 'esfc', 'earc_vrfy', 'earc_vrfy', 'cleanup'] + task_names[run] += ['stage_ic', 'ecen', 'esfc', 'earc_vrfy', 'earc_tars', 'cleanup'] return task_names From bd20dfd16fc40f453669846dd277ccd72e046dcd Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Mon, 20 Jan 2025 17:15:33 -0500 Subject: [PATCH 051/105] directory changed of JJOB --- jobs/JGDAS_ENKF_ARCHIVE_TARS | 44 ++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 jobs/JGDAS_ENKF_ARCHIVE_TARS diff --git a/jobs/JGDAS_ENKF_ARCHIVE_TARS b/jobs/JGDAS_ENKF_ARCHIVE_TARS new file mode 100644 index 0000000000..04fe8e3141 --- /dev/null +++ b/jobs/JGDAS_ENKF_ARCHIVE_TARS @@ -0,0 +1,44 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" +source "${HOMEgfs}/ush/jjob_header.sh" -e "earc_tars" -c "base earc_tars" + + +############################################## +# Set variables used in the script +############################################## +YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_TOP +MEMDIR="ensstat" YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ + COMIN_ATMOS_ANALYSIS_ENSSTAT:COM_ATMOS_ANALYSIS_TMPL \ + COMIN_ATMOS_HISTORY_ENSSTAT:COM_ATMOS_HISTORY_TMPL \ + COMIN_SNOW_ANALYSIS_ENSSTAT:COM_SNOW_ANALYSIS_TMPL + +############################################################### +# Run archive script +############################################################### + +"${SCRgfs}/exgdas_enkf_earc_tars.py" +status=$? +[[ ${status} -ne 0 ]] && exit "${status}" + +############################################################### + +############################################## +# End JOB SPECIFIC work +############################################## + +############################################## +# Final processing +############################################## +if [[ -e "${pgmout}" ]] ; then + cat "${pgmout}" +fi + + +########################################## +# Remove the Temporary working directory +########################################## +cd "${DATAROOT}" || (echo "${DATAROOT} does not exist. ABORT!"; exit 1) +[[ ${KEEPDATA} = "NO" ]] && rm -rf "${DATA}" + +exit 0 From 6a8c09ed4411c253145625cb598784054ed1b4d7 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Mon, 20 Jan 2025 17:16:14 -0500 Subject: [PATCH 052/105] modified jjob --- jobs/rocoto/JGDAS_ENKF_ARCHIVE_TARS | 44 ----------------------------- 1 file changed, 44 deletions(-) delete mode 100644 jobs/rocoto/JGDAS_ENKF_ARCHIVE_TARS diff --git a/jobs/rocoto/JGDAS_ENKF_ARCHIVE_TARS b/jobs/rocoto/JGDAS_ENKF_ARCHIVE_TARS deleted file mode 100644 index 04fe8e3141..0000000000 --- a/jobs/rocoto/JGDAS_ENKF_ARCHIVE_TARS +++ /dev/null @@ -1,44 +0,0 @@ -#! /usr/bin/env bash - -source "${HOMEgfs}/ush/preamble.sh" -source "${HOMEgfs}/ush/jjob_header.sh" -e "earc_tars" -c "base earc_tars" - - -############################################## -# Set variables used in the script -############################################## -YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_TOP -MEMDIR="ensstat" YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ - COMIN_ATMOS_ANALYSIS_ENSSTAT:COM_ATMOS_ANALYSIS_TMPL \ - COMIN_ATMOS_HISTORY_ENSSTAT:COM_ATMOS_HISTORY_TMPL \ - COMIN_SNOW_ANALYSIS_ENSSTAT:COM_SNOW_ANALYSIS_TMPL - -############################################################### -# Run archive script -############################################################### - -"${SCRgfs}/exgdas_enkf_earc_tars.py" -status=$? -[[ ${status} -ne 0 ]] && exit "${status}" - -############################################################### - -############################################## -# End JOB SPECIFIC work -############################################## - -############################################## -# Final processing -############################################## -if [[ -e "${pgmout}" ]] ; then - cat "${pgmout}" -fi - - -########################################## -# Remove the Temporary working directory -########################################## -cd "${DATAROOT}" || (echo "${DATAROOT} does not exist. ABORT!"; exit 1) -[[ ${KEEPDATA} = "NO" ]] && rm -rf "${DATA}" - -exit 0 From 924a8a1d62ce05cd13d8f01d8537bbdbc183cc26 Mon Sep 17 00:00:00 2001 From: Anton Fernando Date: Mon, 20 Jan 2025 22:20:11 +0000 Subject: [PATCH 053/105] change permission --- jobs/JGDAS_ENKF_ARCHIVE_TARS | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 jobs/JGDAS_ENKF_ARCHIVE_TARS diff --git a/jobs/JGDAS_ENKF_ARCHIVE_TARS b/jobs/JGDAS_ENKF_ARCHIVE_TARS old mode 100644 new mode 100755 From fae7f3c98fd5ef05574c5450bda3c2943739521f Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Mon, 20 Jan 2025 22:34:25 -0500 Subject: [PATCH 054/105] modified workflow\rocoto\gfs_tasks.py --- workflow/rocoto/gfs_tasks.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 7051038b18..f76401d40c 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -2442,6 +2442,8 @@ def cleanup(self): if 'enkf' in self.run: dep_dict = {'type': 'metatask', 'name': f'{self.run}_eamn'} deps.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'task', 'name': f'{self.run}_arch_vrfy'} + deps.append(rocoto.add_dependency(dep_dict)) else: dep_dict = {'type': 'task', 'name': f'{self.run}_arch_vrfy'} deps.append(rocoto.add_dependency(dep_dict)) From e0fff9781e747a844e9d7c51a0f66b599bc3281c Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Mon, 20 Jan 2025 22:37:04 -0500 Subject: [PATCH 055/105] modify workflow\rocoto\gfs_tasks.py --- workflow/rocoto/gfs_tasks.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index f76401d40c..7051038b18 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -2442,8 +2442,6 @@ def cleanup(self): if 'enkf' in self.run: dep_dict = {'type': 'metatask', 'name': f'{self.run}_eamn'} deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'task', 'name': f'{self.run}_arch_vrfy'} - deps.append(rocoto.add_dependency(dep_dict)) else: dep_dict = {'type': 'task', 'name': f'{self.run}_arch_vrfy'} deps.append(rocoto.add_dependency(dep_dict)) From 6f2d008475308f5b7e0b86f64b0fe807a7339ea8 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Mon, 20 Jan 2025 22:39:09 -0500 Subject: [PATCH 056/105] modified workflow\rocoto\gfs_tasks.py --- workflow/rocoto/gfs_tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 7051038b18..fa9e21336d 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -2436,7 +2436,7 @@ def arch_tars(self): return task - # Cleanup +# cleanup def cleanup(self): deps = [] if 'enkf' in self.run: From 65571aa573213528532c12200e0e039804e35326 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Tue, 21 Jan 2025 11:55:05 -0500 Subject: [PATCH 057/105] modified workflow\rocoto\gfs_tasks.py --- workflow/rocoto/gfs_tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index fa9e21336d..61b810fa45 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -2442,9 +2442,9 @@ def cleanup(self): if 'enkf' in self.run: dep_dict = {'type': 'metatask', 'name': f'{self.run}_eamn'} deps.append(rocoto.add_dependency(dep_dict)) - else: dep_dict = {'type': 'task', 'name': f'{self.run}_arch_vrfy'} deps.append(rocoto.add_dependency(dep_dict)) + else: if self.options['do_archtar']: dep_dict = {'type': 'task', 'name': f'{self.run}_arch_tars'} deps.append(rocoto.add_dependency(dep_dict)) From c283e87ad6faee86976ba8fa70d59cdb24f879a3 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Tue, 21 Jan 2025 12:05:34 -0500 Subject: [PATCH 058/105] modified config files --- parm/config/gfs/config.earc_tars | 2 +- parm/config/gfs/config.earc_vrfy | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/parm/config/gfs/config.earc_tars b/parm/config/gfs/config.earc_tars index b4a5856118..a3271c6286 100755 --- a/parm/config/gfs/config.earc_tars +++ b/parm/config/gfs/config.earc_tars @@ -6,7 +6,7 @@ echo "BEGIN: config.earc_tars" # Get task specific resources -. $EXPDIR/config.resources earc_tars +. $EXPDIR/config.resources "earc_tars" # Set the number of ensemble members to archive per earc_tars job case "${CASE_ENS}" in diff --git a/parm/config/gfs/config.earc_vrfy b/parm/config/gfs/config.earc_vrfy index 9d1f0cfc19..a00904ecce 100755 --- a/parm/config/gfs/config.earc_vrfy +++ b/parm/config/gfs/config.earc_vrfy @@ -5,8 +5,8 @@ echo "BEGIN: config.earc_vrfy" -# Get task specific resources -. $EXPDIR/config.resources earc_vrfy +# Get task specific resources" +. $EXPDIR/config.resources "earc_vrfy" # Set the number of ensemble members to archive per earc_vrfy job case "${CASE_ENS}" in From 2cc52f32a3e0b39526f0c667f621bb0d0c0541e9 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Tue, 21 Jan 2025 12:17:30 -0500 Subject: [PATCH 059/105] modified config files --- parm/config/gfs/config.earc_tars | 2 +- parm/config/gfs/config.earc_vrfy | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/parm/config/gfs/config.earc_tars b/parm/config/gfs/config.earc_tars index a3271c6286..7d149876af 100755 --- a/parm/config/gfs/config.earc_tars +++ b/parm/config/gfs/config.earc_tars @@ -6,7 +6,7 @@ echo "BEGIN: config.earc_tars" # Get task specific resources -. $EXPDIR/config.resources "earc_tars" +. $EXPDIR/config.resources "${earc_tars}" # Set the number of ensemble members to archive per earc_tars job case "${CASE_ENS}" in diff --git a/parm/config/gfs/config.earc_vrfy b/parm/config/gfs/config.earc_vrfy index a00904ecce..63d0eb5b70 100755 --- a/parm/config/gfs/config.earc_vrfy +++ b/parm/config/gfs/config.earc_vrfy @@ -6,7 +6,7 @@ echo "BEGIN: config.earc_vrfy" # Get task specific resources" -. $EXPDIR/config.resources "earc_vrfy" +. $EXPDIR/config.resources "${earc_vrfy}" # Set the number of ensemble members to archive per earc_vrfy job case "${CASE_ENS}" in From 2657164d691ae7e489434424eed03623f411c48f Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Tue, 21 Jan 2025 12:21:50 -0500 Subject: [PATCH 060/105] modified config files for gfs job --- parm/config/gfs/config.arch_tars | 2 +- parm/config/gfs/config.arch_vrfy | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/parm/config/gfs/config.arch_tars b/parm/config/gfs/config.arch_tars index 7605cc04c1..3a0e29955b 100644 --- a/parm/config/gfs/config.arch_tars +++ b/parm/config/gfs/config.arch_tars @@ -6,7 +6,7 @@ echo "BEGIN: config.arch_tars" # Get task specific resources -. "${EXPDIR}/config.resources" arch_tars +. "${EXPDIR}/config.resources" "${arch_tars}" export ARCH_GAUSSIAN="YES" export ARCH_GAUSSIAN_FHMAX=${FHMAX_GFS} diff --git a/parm/config/gfs/config.arch_vrfy b/parm/config/gfs/config.arch_vrfy index cb668a48e2..ea1d0271a0 100644 --- a/parm/config/gfs/config.arch_vrfy +++ b/parm/config/gfs/config.arch_vrfy @@ -6,7 +6,7 @@ echo "BEGIN: config.arch_vrfy" # Get task specific resources -. "${EXPDIR}/config.resources" arch_vrfy +. "${EXPDIR}/config.resources" "${arch_vrfy} export ARCH_GAUSSIAN="YES" export ARCH_GAUSSIAN_FHMAX=${FHMAX_GFS} From ed807e414ed379349e9cfb34bed8927ec50f1fbe Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Tue, 21 Jan 2025 12:23:18 -0500 Subject: [PATCH 061/105] modified parm\config\gfs\config.arch_vrfy --- parm/config/gfs/config.arch_vrfy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/config/gfs/config.arch_vrfy b/parm/config/gfs/config.arch_vrfy index ea1d0271a0..ad1421ed97 100644 --- a/parm/config/gfs/config.arch_vrfy +++ b/parm/config/gfs/config.arch_vrfy @@ -6,7 +6,7 @@ echo "BEGIN: config.arch_vrfy" # Get task specific resources -. "${EXPDIR}/config.resources" "${arch_vrfy} +. "${EXPDIR}/config.resources" "${arch_vrfy}" export ARCH_GAUSSIAN="YES" export ARCH_GAUSSIAN_FHMAX=${FHMAX_GFS} From cb836f8b6a502473e38bc2f94f919cc7b361667c Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Tue, 21 Jan 2025 12:26:47 -0500 Subject: [PATCH 062/105] modify config files for gfs --- parm/config/gfs/config.earc_tars | 2 +- parm/config/gfs/config.earc_vrfy | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/parm/config/gfs/config.earc_tars b/parm/config/gfs/config.earc_tars index 7d149876af..dd099fbccd 100755 --- a/parm/config/gfs/config.earc_tars +++ b/parm/config/gfs/config.earc_tars @@ -6,7 +6,7 @@ echo "BEGIN: config.earc_tars" # Get task specific resources -. $EXPDIR/config.resources "${earc_tars}" +. "${EXPDIR}/config.resources" "${earc_tars}" # Set the number of ensemble members to archive per earc_tars job case "${CASE_ENS}" in diff --git a/parm/config/gfs/config.earc_vrfy b/parm/config/gfs/config.earc_vrfy index 63d0eb5b70..512649e5a2 100755 --- a/parm/config/gfs/config.earc_vrfy +++ b/parm/config/gfs/config.earc_vrfy @@ -6,7 +6,7 @@ echo "BEGIN: config.earc_vrfy" # Get task specific resources" -. $EXPDIR/config.resources "${earc_vrfy}" +. "${EXPDIR}/config.resources" "${earc_vrfy}" # Set the number of ensemble members to archive per earc_vrfy job case "${CASE_ENS}" in From 1b774661af54d512291618c58b1f33104f60a9ca Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Tue, 21 Jan 2025 13:29:48 -0500 Subject: [PATCH 063/105] added exglobal_archive.py --- scripts/exglobal_archive.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 scripts/exglobal_archive.py diff --git a/scripts/exglobal_archive.py b/scripts/exglobal_archive.py new file mode 100644 index 0000000000..e69de29bb2 From a5966329441f32b169e2e6fb51f1547a07e5a836 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Tue, 21 Jan 2025 13:31:15 -0500 Subject: [PATCH 064/105] added two dleted files to merge with a dev branch --- jobs/JGLOBAL_ARCHIVE | 94 +++++++++++++++++++++++++++++++++++++ scripts/exglobal_archive.py | 74 +++++++++++++++++++++++++++++ 2 files changed, 168 insertions(+) create mode 100644 jobs/JGLOBAL_ARCHIVE diff --git a/jobs/JGLOBAL_ARCHIVE b/jobs/JGLOBAL_ARCHIVE new file mode 100644 index 0000000000..f62386cdd9 --- /dev/null +++ b/jobs/JGLOBAL_ARCHIVE @@ -0,0 +1,94 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" +source "${HOMEgfs}/ush/jjob_header.sh" -e "arch" -c "base arch wave" +source "${USHgfs}/wave_domain_grid.sh" +############################################## +# Set variables used in the script +############################################## +YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ + COMIN_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL \ + COMIN_ATMOS_BUFR:COM_ATMOS_BUFR_TMPL \ + COMIN_ATMOS_GEMPAK:COM_ATMOS_GEMPAK_TMPL \ + COMIN_ATMOS_GENESIS:COM_ATMOS_GENESIS_TMPL \ + COMIN_ATMOS_HISTORY:COM_ATMOS_HISTORY_TMPL \ + COMIN_ATMOS_INPUT:COM_ATMOS_INPUT_TMPL \ + COMIN_ATMOS_MASTER:COM_ATMOS_MASTER_TMPL \ + COMIN_ATMOS_RESTART:COM_ATMOS_RESTART_TMPL \ + COMIN_ATMOS_TRACK:COM_ATMOS_TRACK_TMPL \ + COMIN_ATMOS_WMO:COM_ATMOS_WMO_TMPL \ + COMIN_CHEM_HISTORY:COM_CHEM_HISTORY_TMPL \ + COMIN_CHEM_ANALYSIS:COM_CHEM_ANALYSIS_TMPL \ + COMIN_MED_RESTART:COM_MED_RESTART_TMPL \ + COMIN_SNOW_ANALYSIS:COM_SNOW_ANALYSIS_TMPL \ + COMIN_ICE_HISTORY:COM_ICE_HISTORY_TMPL \ + COMIN_ICE_INPUT:COM_ICE_INPUT_TMPL \ + COMIN_ICE_RESTART:COM_ICE_RESTART_TMPL \ + COMIN_ICE_GRIB:COM_ICE_GRIB_TMPL \ + COMIN_OBS:COM_OBS_TMPL \ + COMIN_TOP:COM_TOP_TMPL \ + COMIN_OCEAN_HISTORY:COM_OCEAN_HISTORY_TMPL \ + COMIN_OCEAN_RESTART:COM_OCEAN_RESTART_TMPL \ + COMIN_OCEAN_GRIB:COM_OCEAN_GRIB_TMPL \ + COMIN_OCEAN_NETCDF:COM_OCEAN_NETCDF_TMPL \ + COMIN_OCEAN_ANALYSIS:COM_OCEAN_ANALYSIS_TMPL \ + COMIN_OCEAN_BMATRIX:COM_OCEAN_BMATRIX_TMPL \ + COMIN_ICE_BMATRIX:COM_ICE_BMATRIX_TMPL \ + COMIN_WAVE_GRID:COM_WAVE_GRID_TMPL \ + COMIN_WAVE_HISTORY:COM_WAVE_HISTORY_TMPL \ + COMIN_WAVE_STATION:COM_WAVE_STATION_TMPL \ + COMIN_WAVE_RESTART:COM_WAVE_RESTART_TMPL \ + COMIN_ATMOS_OZNMON:COM_ATMOS_OZNMON_TMPL \ + COMIN_ATMOS_RADMON:COM_ATMOS_RADMON_TMPL \ + COMIN_ATMOS_MINMON:COM_ATMOS_MINMON_TMPL \ + COMIN_CONF:COM_CONF_TMPL \ + COMOUT_ATMOS_TRACK:COM_ATMOS_TRACK_TMPL + +for grid in "0p25" "0p50" "1p00"; do + YMD=${PDY} HH=${cyc} GRID=${grid} declare_from_tmpl -rx \ + "COMIN_ATMOS_GRIB_${grid}:COM_ATMOS_GRIB_GRID_TMPL" +done + +############################################################### +# Create an array of output wave grids to process +if [[ "${DO_WAVE}" == "YES" ]]; then + # Create a variable for output wave grids to process + if [[ -n "${wavepostGRD}" || -n "${waveinterpGRD}" ]]; then + for grdID in ${wavepostGRD} ${waveinterpGRD}; do + process_grdID "${grdID}" + YMD=${PDY} HH=${cyc} GRDRESNAME=${grdNAME} declare_from_tmpl -rx \ + "COMIN_WAVE_GRID_${GRDREGION}_${GRDRES}:COM_WAVE_GRID_RES_TMPL" + grids_arr+=("${GRDREGION}.${GRDRES}") + done + export WAVE_OUT_GRIDS="${grids_arr[*]}" + else + echo "Both wavepostGRD and waveinterpGRD are empty. No grids to process." + export WAVE_OUT_GRIDS="" + fi +fi +############################################################### +# Run archive script +############################################################### + +${GLOBALARCHIVESH:-${SCRgfs}/exglobal_archive.py} +status=$? +[[ ${status} -ne 0 ]] && exit "${status}" + +############################################## +# End JOB SPECIFIC work +############################################## + +############################################## +# Final processing +############################################## +if [[ -e "${pgmout}" ]] ; then + cat "${pgmout}" +fi + +########################################## +# Remove the Temporary working directory +########################################## +cd "${DATAROOT}" || (echo "${DATAROOT} does not exist. ABORT!"; exit 1) +[[ ${KEEPDATA} = "NO" ]] && rm -rf "${DATA}" + +exit 0 diff --git a/scripts/exglobal_archive.py b/scripts/exglobal_archive.py index e69de29bb2..f64db172ac 100644 --- a/scripts/exglobal_archive.py +++ b/scripts/exglobal_archive.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 + +import os + +from pygfs.task.archive import Archive +from wxflow import AttrDict, Logger, cast_strdict_as_dtypedict, logit, chdir + +# initialize root logger +logger = Logger(level=os.environ.get("LOGGING_LEVEL", "DEBUG"), colored_log=True) + + +@logit(logger) +def main(): + + config = cast_strdict_as_dtypedict(os.environ) + + # Instantiate the Archive object + archive = Archive(config) + + # update these keys to be 3 digits if they are part of archive.task_config.keys + for key in ['OCNRES', 'ICERES']: + try: + archive.task_config[key] = f"{archive.task_config[key]:03d}" + except KeyError as ee: + logger.info(f"key ({key}) not found in archive.task_config!") + + # Pull out all the configuration keys needed to run the rest of archive steps + keys = ['ATARDIR', 'current_cycle', 'FHMIN', 'FHMAX', 'FHOUT', 'RUN', 'PDY', + 'DO_VERFRAD', 'DO_VMINMON', 'DO_VERFOZN', 'DO_ICE', 'DO_PREP_OBS_AERO', + 'PARMgfs', 'DO_OCN', 'DO_WAVE', 'WRITE_DOPOST', 'PSLOT', 'HPSSARCH', 'DO_MOS', + 'DO_JEDISNOWDA', 'LOCALARCH', 'REALTIME', 'ROTDIR', 'ARCH_WARMICFREQ', + 'ARCH_FCSTICFREQ', 'ARCH_CYC', 'assim_freq', 'ARCDIR', 'SDATE', + 'FHMIN_GFS', 'FHMAX_GFS', 'FHOUT_GFS', 'ARCH_GAUSSIAN', 'MODE', + 'FHOUT_OCN', 'FHOUT_ICE', 'FHOUT_OCN_GFS', 'FHOUT_ICE_GFS', 'DO_BUFRSND', 'DOHYBVAR', + 'ARCH_GAUSSIAN_FHMAX', 'ARCH_GAUSSIAN_FHINC', 'ARCH_GAUSSIAN_FHINC', + 'DOIAU', 'OCNRES', 'ICERES', 'NUM_SND_COLLECTIVES', 'FHOUT_WAV', + 'FHOUT_HF_WAV', 'FHMAX_WAV', 'FHMAX_HF_WAV', 'FHMAX_WAV_GFS', + 'restart_interval_gdas', 'restart_interval_gfs', + 'DO_AERO_ANL', 'DO_AERO_FCST', 'DO_CA', 'DOIBP_WAV', 'DO_JEDIOCNVAR', 'DOHYBVAR_OCN', + 'NMEM_ENS', 'DO_JEDIATMVAR', 'DO_VRFY_OCEANDA', 'FHMAX_FITS', 'waveGRD', + 'IAUFHRS', 'DO_FIT2OBS', 'NET', 'FHOUT_HF_GFS', 'FHMAX_HF_GFS', 'REPLAY_ICS', + 'OFFSET_START_HOUR', 'ARCH_EXPDIR', 'EXPDIR', 'ARCH_EXPDIR_FREQ', 'ARCH_HASHES', + 'ARCH_DIFFS', 'SDATE', 'EDATE', 'HOMEgfs', 'DO_GEMPAK', 'WAVE_OUT_GRIDS'] + + archive_dict = AttrDict() + for key in keys: + try: + archive_dict[key] = archive.task_config[key] + except KeyError as ee: + logger.warning(f"WARNING: key ({key}) not found in archive.task_config!") + + # Also import all COMIN* and COMOUT* directory and template variables + for key in archive.task_config.keys(): + if key.startswith(("COM_", "COMIN_", "COMOUT_")): + archive_dict[key] = archive.task_config.get(key) + + with chdir(config.ROTDIR): + + # Determine which archives to create + arcdir_set, atardir_sets = archive.configure(archive_dict) + + # Populate the product archive (ARCDIR) + archive.execute_store_products(arcdir_set) + + # Create the backup tarballs and store in ATARDIR + for atardir_set in atardir_sets: + archive.execute_backup_dataset(atardir_set) + + # Clean up any temporary files + archive.clean() + + +if __name__ == '__main__': + main() From 04679943aad6b743b4dda4faed4492c9885876d2 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Tue, 21 Jan 2025 13:37:47 -0500 Subject: [PATCH 065/105] modified JJOB archive scripts --- jobs/JGLOBAL_ARCHIVE | 94 --------------------------------------- jobs/JGLOBAL_ARCHIVE_TARS | 17 +++++++ jobs/JGLOBAL_ARCHIVE_VRFY | 17 +++++++ 3 files changed, 34 insertions(+), 94 deletions(-) delete mode 100644 jobs/JGLOBAL_ARCHIVE diff --git a/jobs/JGLOBAL_ARCHIVE b/jobs/JGLOBAL_ARCHIVE deleted file mode 100644 index f62386cdd9..0000000000 --- a/jobs/JGLOBAL_ARCHIVE +++ /dev/null @@ -1,94 +0,0 @@ -#! /usr/bin/env bash - -source "${HOMEgfs}/ush/preamble.sh" -source "${HOMEgfs}/ush/jjob_header.sh" -e "arch" -c "base arch wave" -source "${USHgfs}/wave_domain_grid.sh" -############################################## -# Set variables used in the script -############################################## -YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ - COMIN_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL \ - COMIN_ATMOS_BUFR:COM_ATMOS_BUFR_TMPL \ - COMIN_ATMOS_GEMPAK:COM_ATMOS_GEMPAK_TMPL \ - COMIN_ATMOS_GENESIS:COM_ATMOS_GENESIS_TMPL \ - COMIN_ATMOS_HISTORY:COM_ATMOS_HISTORY_TMPL \ - COMIN_ATMOS_INPUT:COM_ATMOS_INPUT_TMPL \ - COMIN_ATMOS_MASTER:COM_ATMOS_MASTER_TMPL \ - COMIN_ATMOS_RESTART:COM_ATMOS_RESTART_TMPL \ - COMIN_ATMOS_TRACK:COM_ATMOS_TRACK_TMPL \ - COMIN_ATMOS_WMO:COM_ATMOS_WMO_TMPL \ - COMIN_CHEM_HISTORY:COM_CHEM_HISTORY_TMPL \ - COMIN_CHEM_ANALYSIS:COM_CHEM_ANALYSIS_TMPL \ - COMIN_MED_RESTART:COM_MED_RESTART_TMPL \ - COMIN_SNOW_ANALYSIS:COM_SNOW_ANALYSIS_TMPL \ - COMIN_ICE_HISTORY:COM_ICE_HISTORY_TMPL \ - COMIN_ICE_INPUT:COM_ICE_INPUT_TMPL \ - COMIN_ICE_RESTART:COM_ICE_RESTART_TMPL \ - COMIN_ICE_GRIB:COM_ICE_GRIB_TMPL \ - COMIN_OBS:COM_OBS_TMPL \ - COMIN_TOP:COM_TOP_TMPL \ - COMIN_OCEAN_HISTORY:COM_OCEAN_HISTORY_TMPL \ - COMIN_OCEAN_RESTART:COM_OCEAN_RESTART_TMPL \ - COMIN_OCEAN_GRIB:COM_OCEAN_GRIB_TMPL \ - COMIN_OCEAN_NETCDF:COM_OCEAN_NETCDF_TMPL \ - COMIN_OCEAN_ANALYSIS:COM_OCEAN_ANALYSIS_TMPL \ - COMIN_OCEAN_BMATRIX:COM_OCEAN_BMATRIX_TMPL \ - COMIN_ICE_BMATRIX:COM_ICE_BMATRIX_TMPL \ - COMIN_WAVE_GRID:COM_WAVE_GRID_TMPL \ - COMIN_WAVE_HISTORY:COM_WAVE_HISTORY_TMPL \ - COMIN_WAVE_STATION:COM_WAVE_STATION_TMPL \ - COMIN_WAVE_RESTART:COM_WAVE_RESTART_TMPL \ - COMIN_ATMOS_OZNMON:COM_ATMOS_OZNMON_TMPL \ - COMIN_ATMOS_RADMON:COM_ATMOS_RADMON_TMPL \ - COMIN_ATMOS_MINMON:COM_ATMOS_MINMON_TMPL \ - COMIN_CONF:COM_CONF_TMPL \ - COMOUT_ATMOS_TRACK:COM_ATMOS_TRACK_TMPL - -for grid in "0p25" "0p50" "1p00"; do - YMD=${PDY} HH=${cyc} GRID=${grid} declare_from_tmpl -rx \ - "COMIN_ATMOS_GRIB_${grid}:COM_ATMOS_GRIB_GRID_TMPL" -done - -############################################################### -# Create an array of output wave grids to process -if [[ "${DO_WAVE}" == "YES" ]]; then - # Create a variable for output wave grids to process - if [[ -n "${wavepostGRD}" || -n "${waveinterpGRD}" ]]; then - for grdID in ${wavepostGRD} ${waveinterpGRD}; do - process_grdID "${grdID}" - YMD=${PDY} HH=${cyc} GRDRESNAME=${grdNAME} declare_from_tmpl -rx \ - "COMIN_WAVE_GRID_${GRDREGION}_${GRDRES}:COM_WAVE_GRID_RES_TMPL" - grids_arr+=("${GRDREGION}.${GRDRES}") - done - export WAVE_OUT_GRIDS="${grids_arr[*]}" - else - echo "Both wavepostGRD and waveinterpGRD are empty. No grids to process." - export WAVE_OUT_GRIDS="" - fi -fi -############################################################### -# Run archive script -############################################################### - -${GLOBALARCHIVESH:-${SCRgfs}/exglobal_archive.py} -status=$? -[[ ${status} -ne 0 ]] && exit "${status}" - -############################################## -# End JOB SPECIFIC work -############################################## - -############################################## -# Final processing -############################################## -if [[ -e "${pgmout}" ]] ; then - cat "${pgmout}" -fi - -########################################## -# Remove the Temporary working directory -########################################## -cd "${DATAROOT}" || (echo "${DATAROOT} does not exist. ABORT!"; exit 1) -[[ ${KEEPDATA} = "NO" ]] && rm -rf "${DATA}" - -exit 0 diff --git a/jobs/JGLOBAL_ARCHIVE_TARS b/jobs/JGLOBAL_ARCHIVE_TARS index 5573a6c778..ee166f152b 100755 --- a/jobs/JGLOBAL_ARCHIVE_TARS +++ b/jobs/JGLOBAL_ARCHIVE_TARS @@ -49,6 +49,23 @@ for grid in "0p25" "0p50" "1p00"; do "COMIN_ATMOS_GRIB_${grid}:COM_ATMOS_GRIB_GRID_TMPL" done +############################################################### +# Create an array of output wave grids to process +if [[ "${DO_WAVE}" == "YES" ]]; then + # Create a variable for output wave grids to process + if [[ -n "${wavepostGRD}" || -n "${waveinterpGRD}" ]]; then + for grdID in ${wavepostGRD} ${waveinterpGRD}; do + process_grdID "${grdID}" + YMD=${PDY} HH=${cyc} GRDRESNAME=${grdNAME} declare_from_tmpl -rx \ + "COMIN_WAVE_GRID_${GRDREGION}_${GRDRES}:COM_WAVE_GRID_RES_TMPL" + grids_arr+=("${GRDREGION}.${GRDRES}") + done + export WAVE_OUT_GRIDS="${grids_arr[*]}" + else + echo "Both wavepostGRD and waveinterpGRD are empty. No grids to process." + export WAVE_OUT_GRIDS="" + fi +fi ############################################################### # Run archive script ############################################################### diff --git a/jobs/JGLOBAL_ARCHIVE_VRFY b/jobs/JGLOBAL_ARCHIVE_VRFY index aba9edf06f..8312df6169 100755 --- a/jobs/JGLOBAL_ARCHIVE_VRFY +++ b/jobs/JGLOBAL_ARCHIVE_VRFY @@ -50,6 +50,23 @@ for grid in "0p25" "0p50" "1p00"; do "COMIN_ATMOS_GRIB_${grid}:COM_ATMOS_GRIB_GRID_TMPL" done +############################################################### +# Create an array of output wave grids to process +if [[ "${DO_WAVE}" == "YES" ]]; then + # Create a variable for output wave grids to process + if [[ -n "${wavepostGRD}" || -n "${waveinterpGRD}" ]]; then + for grdID in ${wavepostGRD} ${waveinterpGRD}; do + process_grdID "${grdID}" + YMD=${PDY} HH=${cyc} GRDRESNAME=${grdNAME} declare_from_tmpl -rx \ + "COMIN_WAVE_GRID_${GRDREGION}_${GRDRES}:COM_WAVE_GRID_RES_TMPL" + grids_arr+=("${GRDREGION}.${GRDRES}") + done + export WAVE_OUT_GRIDS="${grids_arr[*]}" + else + echo "Both wavepostGRD and waveinterpGRD are empty. No grids to process." + export WAVE_OUT_GRIDS="" + fi +fi ############################################################### # Run archive script ############################################################### From 622713cdf9f35f6ac9b8c0b02f4414f57d8272bf Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Tue, 21 Jan 2025 13:43:17 -0500 Subject: [PATCH 066/105] edited exscripts for arch --- scripts/exglobal_archive.py | 74 -------------------------------- scripts/exglobal_archive_tars.py | 2 +- scripts/exglobal_archive_vrfy.py | 2 +- 3 files changed, 2 insertions(+), 76 deletions(-) delete mode 100644 scripts/exglobal_archive.py diff --git a/scripts/exglobal_archive.py b/scripts/exglobal_archive.py deleted file mode 100644 index f64db172ac..0000000000 --- a/scripts/exglobal_archive.py +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env python3 - -import os - -from pygfs.task.archive import Archive -from wxflow import AttrDict, Logger, cast_strdict_as_dtypedict, logit, chdir - -# initialize root logger -logger = Logger(level=os.environ.get("LOGGING_LEVEL", "DEBUG"), colored_log=True) - - -@logit(logger) -def main(): - - config = cast_strdict_as_dtypedict(os.environ) - - # Instantiate the Archive object - archive = Archive(config) - - # update these keys to be 3 digits if they are part of archive.task_config.keys - for key in ['OCNRES', 'ICERES']: - try: - archive.task_config[key] = f"{archive.task_config[key]:03d}" - except KeyError as ee: - logger.info(f"key ({key}) not found in archive.task_config!") - - # Pull out all the configuration keys needed to run the rest of archive steps - keys = ['ATARDIR', 'current_cycle', 'FHMIN', 'FHMAX', 'FHOUT', 'RUN', 'PDY', - 'DO_VERFRAD', 'DO_VMINMON', 'DO_VERFOZN', 'DO_ICE', 'DO_PREP_OBS_AERO', - 'PARMgfs', 'DO_OCN', 'DO_WAVE', 'WRITE_DOPOST', 'PSLOT', 'HPSSARCH', 'DO_MOS', - 'DO_JEDISNOWDA', 'LOCALARCH', 'REALTIME', 'ROTDIR', 'ARCH_WARMICFREQ', - 'ARCH_FCSTICFREQ', 'ARCH_CYC', 'assim_freq', 'ARCDIR', 'SDATE', - 'FHMIN_GFS', 'FHMAX_GFS', 'FHOUT_GFS', 'ARCH_GAUSSIAN', 'MODE', - 'FHOUT_OCN', 'FHOUT_ICE', 'FHOUT_OCN_GFS', 'FHOUT_ICE_GFS', 'DO_BUFRSND', 'DOHYBVAR', - 'ARCH_GAUSSIAN_FHMAX', 'ARCH_GAUSSIAN_FHINC', 'ARCH_GAUSSIAN_FHINC', - 'DOIAU', 'OCNRES', 'ICERES', 'NUM_SND_COLLECTIVES', 'FHOUT_WAV', - 'FHOUT_HF_WAV', 'FHMAX_WAV', 'FHMAX_HF_WAV', 'FHMAX_WAV_GFS', - 'restart_interval_gdas', 'restart_interval_gfs', - 'DO_AERO_ANL', 'DO_AERO_FCST', 'DO_CA', 'DOIBP_WAV', 'DO_JEDIOCNVAR', 'DOHYBVAR_OCN', - 'NMEM_ENS', 'DO_JEDIATMVAR', 'DO_VRFY_OCEANDA', 'FHMAX_FITS', 'waveGRD', - 'IAUFHRS', 'DO_FIT2OBS', 'NET', 'FHOUT_HF_GFS', 'FHMAX_HF_GFS', 'REPLAY_ICS', - 'OFFSET_START_HOUR', 'ARCH_EXPDIR', 'EXPDIR', 'ARCH_EXPDIR_FREQ', 'ARCH_HASHES', - 'ARCH_DIFFS', 'SDATE', 'EDATE', 'HOMEgfs', 'DO_GEMPAK', 'WAVE_OUT_GRIDS'] - - archive_dict = AttrDict() - for key in keys: - try: - archive_dict[key] = archive.task_config[key] - except KeyError as ee: - logger.warning(f"WARNING: key ({key}) not found in archive.task_config!") - - # Also import all COMIN* and COMOUT* directory and template variables - for key in archive.task_config.keys(): - if key.startswith(("COM_", "COMIN_", "COMOUT_")): - archive_dict[key] = archive.task_config.get(key) - - with chdir(config.ROTDIR): - - # Determine which archives to create - arcdir_set, atardir_sets = archive.configure(archive_dict) - - # Populate the product archive (ARCDIR) - archive.execute_store_products(arcdir_set) - - # Create the backup tarballs and store in ATARDIR - for atardir_set in atardir_sets: - archive.execute_backup_dataset(atardir_set) - - # Clean up any temporary files - archive.clean() - - -if __name__ == '__main__': - main() diff --git a/scripts/exglobal_archive_tars.py b/scripts/exglobal_archive_tars.py index e6ae4a41e9..dc8b39050c 100755 --- a/scripts/exglobal_archive_tars.py +++ b/scripts/exglobal_archive_tars.py @@ -40,7 +40,7 @@ def main(): 'NMEM_ENS', 'DO_JEDIATMVAR', 'DO_VRFY_OCEANDA', 'FHMAX_FITS', 'waveGRD', 'IAUFHRS', 'DO_FIT2OBS', 'NET', 'FHOUT_HF_GFS', 'FHMAX_HF_GFS', 'REPLAY_ICS', 'OFFSET_START_HOUR', 'ARCH_EXPDIR', 'EXPDIR', 'ARCH_EXPDIR_FREQ', 'ARCH_HASHES', - 'ARCH_DIFFS', 'SDATE', 'EDATE', 'HOMEgfs', 'DO_GEMPAK'] + 'ARCH_DIFFS', 'SDATE', 'EDATE', 'HOMEgfs', 'DO_GEMPAK', 'WAVE_OUT_GRIDS'] archive_dict = AttrDict() for key in keys: diff --git a/scripts/exglobal_archive_vrfy.py b/scripts/exglobal_archive_vrfy.py index 3315d72386..9decdb4ef9 100755 --- a/scripts/exglobal_archive_vrfy.py +++ b/scripts/exglobal_archive_vrfy.py @@ -40,7 +40,7 @@ def main(): 'NMEM_ENS', 'DO_JEDIATMVAR', 'DO_VRFY_OCEANDA', 'FHMAX_FITS', 'waveGRD', 'IAUFHRS', 'DO_FIT2OBS', 'NET', 'FHOUT_HF_GFS', 'FHMAX_HF_GFS', 'REPLAY_ICS', 'OFFSET_START_HOUR', 'ARCH_EXPDIR', 'EXPDIR', 'ARCH_EXPDIR_FREQ', 'ARCH_HASHES', - 'ARCH_DIFFS', 'SDATE', 'EDATE', 'HOMEgfs', 'DO_GEMPAK'] + 'ARCH_DIFFS', 'SDATE', 'EDATE', 'HOMEgfs', 'DO_GEMPAK', 'WAVE_OUT_GRIDS'] archive_dict = AttrDict() for key in keys: From e37b692015d056c24ca1260002709b592f94ddd9 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Tue, 21 Jan 2025 14:09:03 -0500 Subject: [PATCH 067/105] changed a var to string in config files --- parm/config/gfs/config.arch_tars | 2 +- parm/config/gfs/config.arch_vrfy | 2 +- parm/config/gfs/config.earc_tars | 2 +- parm/config/gfs/config.earc_vrfy | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/parm/config/gfs/config.arch_tars b/parm/config/gfs/config.arch_tars index 3a0e29955b..f46ff45a3a 100644 --- a/parm/config/gfs/config.arch_tars +++ b/parm/config/gfs/config.arch_tars @@ -6,7 +6,7 @@ echo "BEGIN: config.arch_tars" # Get task specific resources -. "${EXPDIR}/config.resources" "${arch_tars}" +. "${EXPDIR}/config.resources" "arch_tars" export ARCH_GAUSSIAN="YES" export ARCH_GAUSSIAN_FHMAX=${FHMAX_GFS} diff --git a/parm/config/gfs/config.arch_vrfy b/parm/config/gfs/config.arch_vrfy index ad1421ed97..6bcbdb57fc 100644 --- a/parm/config/gfs/config.arch_vrfy +++ b/parm/config/gfs/config.arch_vrfy @@ -6,7 +6,7 @@ echo "BEGIN: config.arch_vrfy" # Get task specific resources -. "${EXPDIR}/config.resources" "${arch_vrfy}" +. "${EXPDIR}/config.resources" "arch_vrfy" export ARCH_GAUSSIAN="YES" export ARCH_GAUSSIAN_FHMAX=${FHMAX_GFS} diff --git a/parm/config/gfs/config.earc_tars b/parm/config/gfs/config.earc_tars index dd099fbccd..c87c5e7784 100755 --- a/parm/config/gfs/config.earc_tars +++ b/parm/config/gfs/config.earc_tars @@ -6,7 +6,7 @@ echo "BEGIN: config.earc_tars" # Get task specific resources -. "${EXPDIR}/config.resources" "${earc_tars}" +. "${EXPDIR}/config.resources" "earc_tars" # Set the number of ensemble members to archive per earc_tars job case "${CASE_ENS}" in diff --git a/parm/config/gfs/config.earc_vrfy b/parm/config/gfs/config.earc_vrfy index 512649e5a2..b45b52786f 100755 --- a/parm/config/gfs/config.earc_vrfy +++ b/parm/config/gfs/config.earc_vrfy @@ -6,7 +6,7 @@ echo "BEGIN: config.earc_vrfy" # Get task specific resources" -. "${EXPDIR}/config.resources" "${earc_vrfy}" +. "${EXPDIR}/config.resources" "earc_vrfy" # Set the number of ensemble members to archive per earc_vrfy job case "${CASE_ENS}" in From 6ea928936581e499e5eaa5bdf8871d2c6e7a1c45 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Tue, 21 Jan 2025 14:27:45 -0500 Subject: [PATCH 068/105] changed comments --- docs/source/jobs.rst | 4 ++-- parm/config/gfs/config.base | 2 +- parm/config/gfs/config.resources | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/source/jobs.rst b/docs/source/jobs.rst index 2cdecb01de..73173fad34 100644 --- a/docs/source/jobs.rst +++ b/docs/source/jobs.rst @@ -18,7 +18,7 @@ An experimental run is different from operations in the following ways: * Addition steps in experimental mode: - - archive (arch) + - archive (arch_vrfy & arch_tars) - cleanup (cleanup) @@ -39,7 +39,7 @@ Jobs in the GFS Configuration | analdiag | Creates netCDF diagnostic files containing observation values, innovation (O-F), error, quality control, as well as | | | other analysis-related quantities (cnvstat, radstat, ozstat files). | +-------------------+-----------------------------------------------------------------------------------------------------------------------+ -| arch | Archives select files from the deterministic model and cleans up older data. | +|arch_vrfy/arch_tars| Archives select files from the deterministic model and cleans up older data. | +-------------------+-----------------------------------------------------------------------------------------------------------------------+ | earcN/eamn | Archival script for EnKF: 1) Write select EnKF output to HPSS; 2) Copy select files to online archive; 3) Clean up | | | EnKF temporary run directories; 4) Remove "old" EnKF files from rotating directory. | diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base index 8a79b9ec72..83a120ee3f 100644 --- a/parm/config/gfs/config.base +++ b/parm/config/gfs/config.base @@ -82,7 +82,7 @@ export DO_MOS="NO" # GFS Model Output Statistics - Only su # NO for retrospective parallel; YES for real-time parallel # arch.sh uses REALTIME for MOS. Need to set REALTIME=YES -# if want MOS written to HPSS. Should update arch.sh to +# if want MOS written to HPSS. Should update arch_vrfy.sh and arch_tars to # use RUNMOS flag export REALTIME="YES" diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index 35ef6826c9..6bc355b02b 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -20,8 +20,8 @@ if (( $# != 1 )); then echo "anal sfcanl analcalc analdiag fcst echgres" echo "upp atmos_products" echo "tracker genesis genesis_fsu" - echo "verfozn verfrad vminmon fit2obs metp arch cleanup" - echo "eobs ediag eomg eupd ecen esfc efcs epos earc" + echo "verfozn verfrad vminmon fit2obs metp arc_vrfy arc_tars cleanup" + echo "eobs ediag eomg eupd ecen esfc efcs epos earc_vrfy earc_tars" echo "init_chem mom6ic oceanice_products" echo "waveinit waveprep wavepostsbs wavepostbndpnt wavepostbndpntbll wavepostpnt" echo "wavegempak waveawipsbulls waveawipsgridded" From d87dcebc44f77faa5c147e42202f0bd01c1beaa6 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Tue, 21 Jan 2025 14:29:45 -0500 Subject: [PATCH 069/105] modified workflow\rocoto\gfs_tasks.py --- workflow/rocoto/gfs_tasks.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 61b810fa45..3ba2591e24 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -2444,10 +2444,9 @@ def cleanup(self): deps.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type': 'task', 'name': f'{self.run}_arch_vrfy'} deps.append(rocoto.add_dependency(dep_dict)) - else: - if self.options['do_archtar']: - dep_dict = {'type': 'task', 'name': f'{self.run}_arch_tars'} - deps.append(rocoto.add_dependency(dep_dict)) + if self.options['do_archtar']: + dep_dict = {'type': 'task', 'name': f'{self.run}_arch_tars'} + deps.append(rocoto.add_dependency(dep_dict)) if self.options['do_gempak']: if self.run in ['gdas']: From db51aea976309b9fccbb0760bbf9ded53636f31f Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Tue, 21 Jan 2025 15:18:29 -0500 Subject: [PATCH 070/105] modified workflow\rocoto\gfs_tasks.py --- workflow/rocoto/gfs_tasks.py | 23 ++++------------------- 1 file changed, 4 insertions(+), 19 deletions(-) diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 3ba2591e24..3463c8e9d0 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -3012,22 +3012,12 @@ def earc_vrfy(self): deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - earcenvars = self.envars.copy() - earcenvars.append(rocoto.create_envar(name='ENSGRP', value='#grp#')) - - # Integer division is floor division, but we need ceiling division - n_groups = -(self.nmem // -self._configs['earc_vrfy']['NMEM_EARCGRP']) - groups = ' '.join([f'{grp:02d}' for grp in range(0, n_groups + 1)]) - - resources = self.get_resource('earc_tars') - - var_dict = {'grp': groups} + resources = self.get_resource('earc_vrfy') - task_name = f'{self.run}_earc_vrfy_#grp#' + task_name = f'{self.run}_earc_vrfy' task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': earcenvars, 'cycledef': self.run.replace('enkf', ''), 'command': f'{self.HOMEgfs}/jobs/rocoto/earc_vrfy.sh', 'job_name': f'{self.pslot}_{task_name}_@H', @@ -3035,12 +3025,7 @@ def earc_vrfy(self): 'maxtries': '&MAXTRIES;' } - metatask_dict = {'task_name': f'{self.run}_eamn', - 'var_dict': var_dict, - 'task_dict': task_dict - } - - task = rocoto.create_task(metatask_dict) + task = rocoto.create_task(task_dict) return task @@ -3079,7 +3064,7 @@ def earc_tars(self): 'maxtries': '&MAXTRIES;' } - metatask_dict = {'task_name': f'{self.run}_eamn', + metatask_dict = {'task_name': f'{self.run}_earc_tars', 'var_dict': var_dict, 'task_dict': task_dict } From 1946c579d630e302448cb766f3e5a7f7147cdcb1 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Tue, 21 Jan 2025 15:33:33 -0500 Subject: [PATCH 071/105] modified workflow\applications\gfs_cycled.py --- workflow/applications/gfs_cycled.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index 3fdb812659..509cdb62e4 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -84,8 +84,14 @@ def _get_app_configs(self, run): 'atmensanlletkf', 'atmensanlfv3inc', 'atmensanlfinal'] else: configs += ['eobs', 'eomg', 'ediag', 'eupd'] + + if options['do_arctar']: + configs += ['earc_tars'] + configs += ['ecen', 'esfc', 'efcs', 'echgres', 'epos', 'earc_vrfy', 'earc_tars'] + + if options['do_fit2obs']: configs += ['fit2obs'] @@ -323,6 +329,9 @@ def get_task_names(self): task_names[run].append('efcs') if 'gdas' in run else 0 task_names[run].append('epos') if 'gdas' in run else 0 - task_names[run] += ['stage_ic', 'ecen', 'esfc', 'earc_vrfy', 'earc_tars', 'cleanup'] + + if options['do_arctars']: + task_names[run] += ['earc_tars'] + task_names[run] += ['stage_ic', 'ecen', 'esfc', 'earc_vrfy', 'cleanup'] return task_names From c1a51c62332341307f252407274e29b5262e1640 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Tue, 21 Jan 2025 15:36:05 -0500 Subject: [PATCH 072/105] modified workflow\applications\gfs_cycled.py --- workflow/applications/gfs_cycled.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index 509cdb62e4..4426476331 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -90,8 +90,6 @@ def _get_app_configs(self, run): configs += ['ecen', 'esfc', 'efcs', 'echgres', 'epos', 'earc_vrfy', 'earc_tars'] - - if options['do_fit2obs']: configs += ['fit2obs'] From a5b63fa416403c933d6848c6d84c058895343cec Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Tue, 21 Jan 2025 15:44:42 -0500 Subject: [PATCH 073/105] added a blank line --- workflow/rocoto/gfs_tasks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 3463c8e9d0..444a9f41b0 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -2444,6 +2444,7 @@ def cleanup(self): deps.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type': 'task', 'name': f'{self.run}_arch_vrfy'} deps.append(rocoto.add_dependency(dep_dict)) + if self.options['do_archtar']: dep_dict = {'type': 'task', 'name': f'{self.run}_arch_tars'} deps.append(rocoto.add_dependency(dep_dict)) From 755ba7c94a7335f0902a9a7671a47cd2c81976dc Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Tue, 21 Jan 2025 15:49:00 -0500 Subject: [PATCH 074/105] modified workflow\applications\gfs_cycled.py --- workflow/applications/gfs_cycled.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index 4426476331..7c8600ca5e 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -85,7 +85,7 @@ def _get_app_configs(self, run): else: configs += ['eobs', 'eomg', 'ediag', 'eupd'] - if options['do_arctar']: + if options['do_archtar']: configs += ['earc_tars'] configs += ['ecen', 'esfc', 'efcs', 'echgres', 'epos', 'earc_vrfy', 'earc_tars'] @@ -328,7 +328,7 @@ def get_task_names(self): task_names[run].append('efcs') if 'gdas' in run else 0 task_names[run].append('epos') if 'gdas' in run else 0 - if options['do_arctars']: + if options['do_archtar']: task_names[run] += ['earc_tars'] task_names[run] += ['stage_ic', 'ecen', 'esfc', 'earc_vrfy', 'cleanup'] From 58e314098d5f5d4ffb9d775e331e380b57dd7818 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Tue, 21 Jan 2025 16:04:41 -0500 Subject: [PATCH 075/105] modify workflow\rocoto\gfs_tasks.py --- workflow/rocoto/gfs_tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 444a9f41b0..54cb984ec4 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -2440,7 +2440,7 @@ def arch_tars(self): def cleanup(self): deps = [] if 'enkf' in self.run: - dep_dict = {'type': 'metatask', 'name': f'{self.run}_eamn'} + dep_dict = {'type': 'metatask', 'name': f'{self.run}_earc_tars'} deps.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type': 'task', 'name': f'{self.run}_arch_vrfy'} deps.append(rocoto.add_dependency(dep_dict)) From 93baf25d375cab9ddeb03930e943bf6460890a37 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Wed, 22 Jan 2025 12:48:29 -0500 Subject: [PATCH 076/105] modifed earc_vrfy task --- workflow/applications/gfs_cycled.py | 7 +++++-- workflow/rocoto/gfs_tasks.py | 4 ++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index 7c8600ca5e..80bda9bbe9 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -78,6 +78,9 @@ def _get_app_configs(self, run): configs += ['stage_ic', 'sfcanl', 'analcalc', 'fcst', 'upp', 'atmos_products', 'arch_vrfy', 'cleanup'] + if options['do_archtar']: + configs += ['arch_tars'] + if options['do_hybvar']: if options['do_jediatmens']: configs += ['atmensanlinit', 'atmensanlobs', 'atmensanlsol', @@ -85,11 +88,11 @@ def _get_app_configs(self, run): else: configs += ['eobs', 'eomg', 'ediag', 'eupd'] + configs += ['ecen', 'esfc', 'efcs', 'echgres', 'epos', 'earc_vrfy'] + if options['do_archtar']: configs += ['earc_tars'] - configs += ['ecen', 'esfc', 'efcs', 'echgres', 'epos', 'earc_vrfy', 'earc_tars'] - if options['do_fit2obs']: configs += ['fit2obs'] diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index df3c3a7ce9..4ed26ec226 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -3013,12 +3013,16 @@ def earc_vrfy(self): deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + earcenvars = self.envars.copy() + earcenvars.append(rocoto.create_envar(name='ENSGRP', value='#grp#')) + resources = self.get_resource('earc_vrfy') task_name = f'{self.run}_earc_vrfy' task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, + 'envars': earcenvars, 'cycledef': self.run.replace('enkf', ''), 'command': f'{self.HOMEgfs}/jobs/rocoto/earc_vrfy.sh', 'job_name': f'{self.pslot}_{task_name}_@H', From bc38f081539e390606840dfb8cff8a688524ff93 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Wed, 22 Jan 2025 13:00:18 -0500 Subject: [PATCH 077/105] removed white space --- workflow/applications/gfs_cycled.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index 80bda9bbe9..15494f289f 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -79,7 +79,7 @@ def _get_app_configs(self, run): configs += ['stage_ic', 'sfcanl', 'analcalc', 'fcst', 'upp', 'atmos_products', 'arch_vrfy', 'cleanup'] if options['do_archtar']: - configs += ['arch_tars'] + configs += ['arch_tars'] if options['do_hybvar']: if options['do_jediatmens']: From 6764985da90ab60ff467c502ee64f09538cb4128 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Wed, 22 Jan 2025 13:22:44 -0500 Subject: [PATCH 078/105] modified workflow\rocoto\gfs_tasks.py --- workflow/rocoto/gfs_tasks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 4ed26ec226..3ee72b712a 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -2442,6 +2442,7 @@ def cleanup(self): if 'enkf' in self.run: dep_dict = {'type': 'metatask', 'name': f'{self.run}_earc_tars'} deps.append(rocoto.add_dependency(dep_dict)) + else: dep_dict = {'type': 'task', 'name': f'{self.run}_arch_vrfy'} deps.append(rocoto.add_dependency(dep_dict)) From 04c20d060d294eaa76ff7111bdedb335fc4227c5 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Wed, 22 Jan 2025 13:28:01 -0500 Subject: [PATCH 079/105] modified workflow\rocoto\gfs_tasks.py --- workflow/rocoto/gfs_tasks.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 3ee72b712a..0771961ccc 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -2440,15 +2440,18 @@ def arch_tars(self): def cleanup(self): deps = [] if 'enkf' in self.run: - dep_dict = {'type': 'metatask', 'name': f'{self.run}_earc_tars'} + dep_dict = {'type': 'metatask', 'name': f'{self.run}_earc_vrfy'} deps.append(rocoto.add_dependency(dep_dict)) + if self.options['do_archtar']: + dep_dict = {'type': 'metatask', 'name': f'{self.run}_earc_tars'} + deps.append(rocoto.add_dependency(dep_dict)) + else: dep_dict = {'type': 'task', 'name': f'{self.run}_arch_vrfy'} deps.append(rocoto.add_dependency(dep_dict)) - - if self.options['do_archtar']: - dep_dict = {'type': 'task', 'name': f'{self.run}_arch_tars'} - deps.append(rocoto.add_dependency(dep_dict)) + if self.options['do_archtar']: + dep_dict = {'type': 'task', 'name': f'{self.run}_arch_tars'} + deps.append(rocoto.add_dependency(dep_dict)) if self.options['do_gempak']: if self.run in ['gdas']: From 7e16fcc403a299c5433e26d274dc4deb5a2bf7db Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Wed, 22 Jan 2025 13:51:48 -0500 Subject: [PATCH 080/105] modified workflow\rocoto\gfs_tasks.py --- workflow/rocoto/gfs_tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 0771961ccc..ccde2cc5be 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -2440,7 +2440,7 @@ def arch_tars(self): def cleanup(self): deps = [] if 'enkf' in self.run: - dep_dict = {'type': 'metatask', 'name': f'{self.run}_earc_vrfy'} + dep_dict = {'type': 'task', 'name': f'{self.run}_earc_vrfy'} deps.append(rocoto.add_dependency(dep_dict)) if self.options['do_archtar']: dep_dict = {'type': 'metatask', 'name': f'{self.run}_earc_tars'} From 4036f67b579eb58333cdcee64ffa4ed141a9e90c Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Wed, 22 Jan 2025 13:57:46 -0500 Subject: [PATCH 081/105] modified workflow\applications\gfs_cycled.py --- workflow/applications/gfs_cycled.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index 15494f289f..db8b4eca79 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -153,9 +153,6 @@ def _get_app_configs(self, run): 'mos_stn_prdgen', 'mos_grd_prdgen', 'mos_ext_stn_prdgen', 'mos_ext_grd_prdgen', 'mos_wx_prdgen', 'mos_wx_ext_prdgen'] - if options['do_archtar']: - configs += ['arch_tars'] - return configs @staticmethod From 604b026c12e81be776a840bf195cd1d1f969becb Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Wed, 22 Jan 2025 14:03:44 -0500 Subject: [PATCH 082/105] modified workflow\applications\gfs_cycled.py --- workflow/applications/gfs_cycled.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index db8b4eca79..8689022414 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -301,10 +301,9 @@ def get_task_names(self): 'mos_stn_prdgen', 'mos_grd_prdgen', 'mos_ext_stn_prdgen', 'mos_ext_grd_prdgen', 'mos_wx_prdgen', 'mos_wx_ext_prdgen'] - if options['do_archtar']: - task_names[run] += ['arch_tars'] - # Last two items + if options['do_archtar']: + task_names[run] += ['arch_tars'] task_names[run] += ['arch_vrfy', 'cleanup'] # Ensemble tasks From e9969dc4a2204424ce5a8516f4be25ebebda5bd8 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Wed, 22 Jan 2025 14:06:47 -0500 Subject: [PATCH 083/105] modified workflow\applications\gfs_cycled.py --- workflow/applications/gfs_cycled.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index 8689022414..6962e97263 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -327,8 +327,9 @@ def get_task_names(self): task_names[run].append('efcs') if 'gdas' in run else 0 task_names[run].append('epos') if 'gdas' in run else 0 + task_names[run] += ['stage_ic', 'ecen', 'esfc'] if options['do_archtar']: task_names[run] += ['earc_tars'] - task_names[run] += ['stage_ic', 'ecen', 'esfc', 'earc_vrfy', 'cleanup'] + task_names[run] += ['earc_vrfy', 'cleanup'] return task_names From c5e611bba2b90cac2be994fb8eaf246a4b8a6951 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Thu, 23 Jan 2025 15:36:01 -0500 Subject: [PATCH 084/105] Walters suggestions --- docs/source/jobs.rst | 4 ++- jobs/JGLOBAL_ARCHIVE_VRFY | 46 ------------------------------- parm/config/gefs/config.base | 2 -- parm/config/gefs/config.resources | 10 ++++++- parm/config/gfs/config.resources | 10 ++++++- scripts/exgdas_enkf_earc_vrfy.py | 13 ++------- scripts/exglobal_archive_vrfy.py | 19 ++----------- ush/python/pygfs/task/archive.py | 2 +- 8 files changed, 27 insertions(+), 79 deletions(-) diff --git a/docs/source/jobs.rst b/docs/source/jobs.rst index 73173fad34..c5f45b9659 100644 --- a/docs/source/jobs.rst +++ b/docs/source/jobs.rst @@ -39,7 +39,9 @@ Jobs in the GFS Configuration | analdiag | Creates netCDF diagnostic files containing observation values, innovation (O-F), error, quality control, as well as | | | other analysis-related quantities (cnvstat, radstat, ozstat files). | +-------------------+-----------------------------------------------------------------------------------------------------------------------+ -|arch_vrfy/arch_tars| Archives select files from the deterministic model and cleans up older data. | +|arch_tars | Optional archive job that backs up the COM data structure. | ++-------------------+-----------------------------------------------------------------------------------------------------------------------+ +|arch_vrfy | Archives select files from the deterministic model and cleans up older data. | +-------------------+-----------------------------------------------------------------------------------------------------------------------+ | earcN/eamn | Archival script for EnKF: 1) Write select EnKF output to HPSS; 2) Copy select files to online archive; 3) Clean up | | | EnKF temporary run directories; 4) Remove "old" EnKF files from rotating directory. | diff --git a/jobs/JGLOBAL_ARCHIVE_VRFY b/jobs/JGLOBAL_ARCHIVE_VRFY index 8312df6169..03525171ba 100755 --- a/jobs/JGLOBAL_ARCHIVE_VRFY +++ b/jobs/JGLOBAL_ARCHIVE_VRFY @@ -9,64 +9,18 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "arch_vrfy" -c "base arch_vrfy" ############################################## YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ COMIN_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL \ - COMIN_ATMOS_BUFR:COM_ATMOS_BUFR_TMPL \ - COMIN_ATMOS_GEMPAK:COM_ATMOS_GEMPAK_TMPL \ COMIN_ATMOS_GENESIS:COM_ATMOS_GENESIS_TMPL \ COMIN_ATMOS_HISTORY:COM_ATMOS_HISTORY_TMPL \ - COMIN_ATMOS_INPUT:COM_ATMOS_INPUT_TMPL \ - COMIN_ATMOS_MASTER:COM_ATMOS_MASTER_TMPL \ - COMIN_ATMOS_RESTART:COM_ATMOS_RESTART_TMPL \ COMIN_ATMOS_TRACK:COM_ATMOS_TRACK_TMPL \ - COMIN_ATMOS_WMO:COM_ATMOS_WMO_TMPL \ - COMIN_CHEM_HISTORY:COM_CHEM_HISTORY_TMPL \ COMIN_CHEM_ANALYSIS:COM_CHEM_ANALYSIS_TMPL \ - COMIN_MED_RESTART:COM_MED_RESTART_TMPL \ COMIN_SNOW_ANALYSIS:COM_SNOW_ANALYSIS_TMPL \ - COMIN_ICE_HISTORY:COM_ICE_HISTORY_TMPL \ - COMIN_ICE_INPUT:COM_ICE_INPUT_TMPL \ - COMIN_ICE_RESTART:COM_ICE_RESTART_TMPL \ - COMIN_ICE_GRIB:COM_ICE_GRIB_TMPL \ COMIN_OBS:COM_OBS_TMPL \ - COMIN_TOP:COM_TOP_TMPL \ - COMIN_OCEAN_HISTORY:COM_OCEAN_HISTORY_TMPL \ - COMIN_OCEAN_RESTART:COM_OCEAN_RESTART_TMPL \ - COMIN_OCEAN_GRIB:COM_OCEAN_GRIB_TMPL \ - COMIN_OCEAN_NETCDF:COM_OCEAN_NETCDF_TMPL \ - COMIN_OCEAN_ANALYSIS:COM_OCEAN_ANALYSIS_TMPL \ - COMIN_OCEAN_BMATRIX:COM_OCEAN_BMATRIX_TMPL \ - COMIN_ICE_BMATRIX:COM_ICE_BMATRIX_TMPL \ - COMIN_WAVE_GRID:COM_WAVE_GRID_TMPL \ - COMIN_WAVE_HISTORY:COM_WAVE_HISTORY_TMPL \ - COMIN_WAVE_STATION:COM_WAVE_STATION_TMPL \ - COMIN_WAVE_RESTART:COM_WAVE_RESTART_TMPL \ - COMIN_ATMOS_OZNMON:COM_ATMOS_OZNMON_TMPL \ - COMIN_ATMOS_RADMON:COM_ATMOS_RADMON_TMPL \ - COMIN_ATMOS_MINMON:COM_ATMOS_MINMON_TMPL \ - COMIN_CONF:COM_CONF_TMPL \ - COMOUT_ATMOS_TRACK:COM_ATMOS_TRACK_TMPL for grid in "0p25" "0p50" "1p00"; do YMD=${PDY} HH=${cyc} GRID=${grid} declare_from_tmpl -rx \ "COMIN_ATMOS_GRIB_${grid}:COM_ATMOS_GRIB_GRID_TMPL" done -############################################################### -# Create an array of output wave grids to process -if [[ "${DO_WAVE}" == "YES" ]]; then - # Create a variable for output wave grids to process - if [[ -n "${wavepostGRD}" || -n "${waveinterpGRD}" ]]; then - for grdID in ${wavepostGRD} ${waveinterpGRD}; do - process_grdID "${grdID}" - YMD=${PDY} HH=${cyc} GRDRESNAME=${grdNAME} declare_from_tmpl -rx \ - "COMIN_WAVE_GRID_${GRDREGION}_${GRDRES}:COM_WAVE_GRID_RES_TMPL" - grids_arr+=("${GRDREGION}.${GRDRES}") - done - export WAVE_OUT_GRIDS="${grids_arr[*]}" - else - echo "Both wavepostGRD and waveinterpGRD are empty. No grids to process." - export WAVE_OUT_GRIDS="" - fi -fi ############################################################### # Run archive script ############################################################### diff --git a/parm/config/gefs/config.base b/parm/config/gefs/config.base index cbcba0cde7..e1cd1928e3 100644 --- a/parm/config/gefs/config.base +++ b/parm/config/gefs/config.base @@ -352,8 +352,6 @@ export ARCH_EXPDIR_FREQ=0 # How often to archive the EXPDIR in hours or 0 for export ARCH_HASHES='YES' # Archive the hashes of the GW and submodules and 'git status' for each; requires ARCH_EXPDIR export ARCH_DIFFS='NO' # Archive the output of 'git diff' for the GW; requires ARCH_EXPDIR -export DELETE_COM_IN_ARCHIVE_JOB="YES" # NO=retain ROTDIR. YES default in arch_*.sh and earc_*.sh. - # Number of regional collectives to create soundings for export NUM_SND_COLLECTIVES=${NUM_SND_COLLECTIVES:-9} diff --git a/parm/config/gefs/config.resources b/parm/config/gefs/config.resources index 7f24a080ac..9c6522a992 100644 --- a/parm/config/gefs/config.resources +++ b/parm/config/gefs/config.resources @@ -323,7 +323,7 @@ case ${step} in export is_exclusive=False ;; - "arch_vrfy" | "arch_tars") + "arch_tars") export walltime="06:00:00" export ntasks=1 export tasks_per_node=1 @@ -331,6 +331,14 @@ case ${step} in export memory="4096M" ;; + "arch_vrfy") + export walltime="00:10:00" + export ntasks=1 + export tasks_per_node=1 + export threads_per_task=1 + export memory="4096M" + ;; + "cleanup") export walltime="00:30:00" export ntasks=1 diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index 95229a066b..8753fa37dd 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -1059,7 +1059,7 @@ case ${step} in export is_exclusive=True ;; - "arch_vrfy" | "arch_tars" | "earc_vrfy" | "earc_tars" | "getic") + "arch_tars" | "earc_vrfy" | "earc_tars" | "getic") walltime="06:00:00" ntasks=1 tasks_per_node=1 @@ -1067,6 +1067,14 @@ case ${step} in memory="4096M" ;; + "arch_vrfy") + walltime="00:15:00" + ntasks=1 + tasks_per_node=1 + threads_per_task=1 + memory="4096M" + ;; + "cleanup") walltime="00:15:00" ntasks=1 diff --git a/scripts/exgdas_enkf_earc_vrfy.py b/scripts/exgdas_enkf_earc_vrfy.py index da893c13ae..1c951f44cb 100755 --- a/scripts/exgdas_enkf_earc_vrfy.py +++ b/scripts/exgdas_enkf_earc_vrfy.py @@ -18,17 +18,8 @@ def main(): archive = Archive(config) # Pull out all the configuration keys needed to run the rest of archive steps - keys = ['ATARDIR', 'current_cycle', 'IAUFHRS', 'RUN', 'PDY', - 'PSLOT', 'HPSSARCH', 'LOCALARCH', 'ROTDIR', 'PARMgfs', - 'ARCDIR', 'SDATE', 'MODE', 'ENSGRP', 'NMEM_EARCGRP', - 'NMEM_ENS', 'DO_CALC_INCREMENT_ENKF_GFS', 'DO_JEDIATMENS', - 'lobsdiag_forenkf', 'FHMIN_ENKF', 'FHMAX_ENKF_GFS', - 'FHOUT_ENKF_GFS', 'FHMAX_ENKF', 'FHOUT_ENKF', 'ENKF_SPREAD', - 'restart_interval_enkfgdas', 'restart_interval_enkfgfs', - 'DOHYBVAR', 'DOIAU_ENKF', 'IAU_OFFSET', 'DOIAU', 'DO_CA', - 'DO_CALC_INCREMENT', 'assim_freq', 'ARCH_CYC', 'DO_JEDISNOWDA', - 'ARCH_WARMICFREQ', 'ARCH_FCSTICFREQ', - 'IAUFHRS_ENKF', 'NET', 'NMEM_ENS_GFS'] + keys = ['current_cycle', 'RUN', 'PSLOT', 'ROTDIR', 'PARMgfs', + 'ARCDIR', 'MODE', 'DO_JEDIATMENS', 'DO_FIT2OBS'] archive_dict = AttrDict() for key in keys: diff --git a/scripts/exglobal_archive_vrfy.py b/scripts/exglobal_archive_vrfy.py index 9decdb4ef9..ae7e0d7de1 100755 --- a/scripts/exglobal_archive_vrfy.py +++ b/scripts/exglobal_archive_vrfy.py @@ -25,22 +25,9 @@ def main(): logger.info(f"key ({key}) not found in archive.task_config!") # Pull out all the configuration keys needed to run the rest of archive steps - keys = ['ATARDIR', 'current_cycle', 'FHMIN', 'FHMAX', 'FHOUT', 'RUN', 'PDY', - 'DO_VERFRAD', 'DO_VMINMON', 'DO_VERFOZN', 'DO_ICE', 'DO_PREP_OBS_AERO', - 'PARMgfs', 'DO_OCN', 'DO_WAVE', 'WRITE_DOPOST', 'PSLOT', 'HPSSARCH', 'DO_MOS', - 'DO_JEDISNOWDA', 'LOCALARCH', 'REALTIME', 'ROTDIR', 'ARCH_WARMICFREQ', - 'ARCH_FCSTICFREQ', 'ARCH_CYC', 'assim_freq', 'ARCDIR', 'SDATE', - 'FHMIN_GFS', 'FHMAX_GFS', 'FHOUT_GFS', 'ARCH_GAUSSIAN', 'MODE', - 'FHOUT_OCN', 'FHOUT_ICE', 'FHOUT_OCN_GFS', 'FHOUT_ICE_GFS', 'DO_BUFRSND', 'DOHYBVAR', - 'ARCH_GAUSSIAN_FHMAX', 'ARCH_GAUSSIAN_FHINC', 'ARCH_GAUSSIAN_FHINC', - 'DOIAU', 'OCNRES', 'ICERES', 'NUM_SND_COLLECTIVES', 'FHOUT_WAV', - 'FHOUT_HF_WAV', 'FHMAX_WAV', 'FHMAX_HF_WAV', 'FHMAX_WAV_GFS', - 'restart_interval_gdas', 'restart_interval_gfs', 'DO_ARCHTAR', - 'DO_AERO_ANL', 'DO_AERO_FCST', 'DO_CA', 'DOIBP_WAV', 'DO_JEDIOCNVAR', 'DOHYBVAR_OCN', - 'NMEM_ENS', 'DO_JEDIATMVAR', 'DO_VRFY_OCEANDA', 'FHMAX_FITS', 'waveGRD', - 'IAUFHRS', 'DO_FIT2OBS', 'NET', 'FHOUT_HF_GFS', 'FHMAX_HF_GFS', 'REPLAY_ICS', - 'OFFSET_START_HOUR', 'ARCH_EXPDIR', 'EXPDIR', 'ARCH_EXPDIR_FREQ', 'ARCH_HASHES', - 'ARCH_DIFFS', 'SDATE', 'EDATE', 'HOMEgfs', 'DO_GEMPAK', 'WAVE_OUT_GRIDS'] + keys = ['current_cycle', 'RUN', 'PSLOT', 'ROTDIR', 'PARMgfs', + 'ARCDIR', 'MODE', 'DO_JEDIATMENS', 'DO_FIT2OBS', 'DO_JEDIATMVAR' + 'DO_JEDISNOWDA', 'DO_AERO_ANL', 'DO_PREP_OBS_AERO'] archive_dict = AttrDict() for key in keys: diff --git a/ush/python/pygfs/task/archive.py b/ush/python/pygfs/task/archive.py index 4d5bac931f..2620bc840b 100644 --- a/ush/python/pygfs/task/archive.py +++ b/ush/python/pygfs/task/archive.py @@ -49,7 +49,7 @@ def __init__(self, config: Dict[str, Any]) -> None: @logit(logger) def configure_vrfy(self, arch_dict: Dict[str, Any]) -> (Dict[str, Any]): - """Determine which files will need to be created to archive tp arcdir. + """Determine which files will need to be created to archive to arcdir. Parameters ---------- From 19fab0e26ecbe3fa150a042d57134ad8e39478b6 Mon Sep 17 00:00:00 2001 From: Anton Fernando Date: Thu, 23 Jan 2025 21:02:01 +0000 Subject: [PATCH 085/105] permission change --- jobs/JGDAS_ENKF_ARCHIVE_VRFY | 0 jobs/JGLOBAL_ARCHIVE_TARS | 0 jobs/JGLOBAL_ARCHIVE_VRFY | 0 jobs/rocoto/arch_tars.sh | 0 jobs/rocoto/arch_vrfy.sh | 0 parm/config/gefs/config.arch_tars | 0 parm/config/gfs/config.earc_tars | 0 parm/config/gfs/config.earc_vrfy | 0 scripts/exgdas_enkf_earc_tars.py | 0 scripts/exgdas_enkf_earc_vrfy.py | 0 scripts/exglobal_archive_tars.py | 0 11 files changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 jobs/JGDAS_ENKF_ARCHIVE_VRFY mode change 100755 => 100644 jobs/JGLOBAL_ARCHIVE_TARS mode change 100755 => 100644 jobs/JGLOBAL_ARCHIVE_VRFY mode change 100755 => 100644 jobs/rocoto/arch_tars.sh mode change 100755 => 100644 jobs/rocoto/arch_vrfy.sh mode change 100755 => 100644 parm/config/gefs/config.arch_tars mode change 100755 => 100644 parm/config/gfs/config.earc_tars mode change 100755 => 100644 parm/config/gfs/config.earc_vrfy mode change 100755 => 100644 scripts/exgdas_enkf_earc_tars.py mode change 100755 => 100644 scripts/exgdas_enkf_earc_vrfy.py mode change 100755 => 100644 scripts/exglobal_archive_tars.py diff --git a/jobs/JGDAS_ENKF_ARCHIVE_VRFY b/jobs/JGDAS_ENKF_ARCHIVE_VRFY old mode 100755 new mode 100644 diff --git a/jobs/JGLOBAL_ARCHIVE_TARS b/jobs/JGLOBAL_ARCHIVE_TARS old mode 100755 new mode 100644 diff --git a/jobs/JGLOBAL_ARCHIVE_VRFY b/jobs/JGLOBAL_ARCHIVE_VRFY old mode 100755 new mode 100644 diff --git a/jobs/rocoto/arch_tars.sh b/jobs/rocoto/arch_tars.sh old mode 100755 new mode 100644 diff --git a/jobs/rocoto/arch_vrfy.sh b/jobs/rocoto/arch_vrfy.sh old mode 100755 new mode 100644 diff --git a/parm/config/gefs/config.arch_tars b/parm/config/gefs/config.arch_tars old mode 100755 new mode 100644 diff --git a/parm/config/gfs/config.earc_tars b/parm/config/gfs/config.earc_tars old mode 100755 new mode 100644 diff --git a/parm/config/gfs/config.earc_vrfy b/parm/config/gfs/config.earc_vrfy old mode 100755 new mode 100644 diff --git a/scripts/exgdas_enkf_earc_tars.py b/scripts/exgdas_enkf_earc_tars.py old mode 100755 new mode 100644 diff --git a/scripts/exgdas_enkf_earc_vrfy.py b/scripts/exgdas_enkf_earc_vrfy.py old mode 100755 new mode 100644 diff --git a/scripts/exglobal_archive_tars.py b/scripts/exglobal_archive_tars.py old mode 100755 new mode 100644 From 5814f59d5f4d9ecb31015d382cd88fa388426382 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Fri, 24 Jan 2025 11:26:19 -0500 Subject: [PATCH 086/105] modified workflow\rocoto\gfs_tasks.py --- workflow/rocoto/gfs_tasks.py | 29 ----------------------------- 1 file changed, 29 deletions(-) diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index ccde2cc5be..75cbe66931 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -2280,24 +2280,12 @@ def arch_vrfy(self): if self.run in ['gfs']: dep_dict = {'type': 'task', 'name': f'{self.run}_atmanlprod'} deps.append(rocoto.add_dependency(dep_dict)) - if self.options['do_vminmon']: - dep_dict = {'type': 'task', 'name': f'{self.run}_vminmon'} - deps.append(rocoto.add_dependency(dep_dict)) elif self.run in ['gdas']: dep_dict = {'type': 'task', 'name': f'{self.run}_atmanlprod'} deps.append(rocoto.add_dependency(dep_dict)) if self.options['do_fit2obs']: dep_dict = {'type': 'task', 'name': f'{self.run}_fit2obs'} deps.append(rocoto.add_dependency(dep_dict)) - if self.options['do_verfozn']: - dep_dict = {'type': 'task', 'name': f'{self.run}_verfozn'} - deps.append(rocoto.add_dependency(dep_dict)) - if self.options['do_verfrad']: - dep_dict = {'type': 'task', 'name': f'{self.run}_verfrad'} - deps.append(rocoto.add_dependency(dep_dict)) - if self.options['do_vminmon']: - dep_dict = {'type': 'task', 'name': f'{self.run}_vminmon'} - deps.append(rocoto.add_dependency(dep_dict)) if self.run in ['gfs'] and self.options['do_tracker']: dep_dict = {'type': 'task', 'name': f'{self.run}_tracker'} deps.append(rocoto.add_dependency(dep_dict)) @@ -2310,14 +2298,6 @@ def arch_vrfy(self): # Post job dependencies dep_dict = {'type': 'metatask', 'name': f'{self.run}_atmos_prod'} deps.append(rocoto.add_dependency(dep_dict)) - if self.options['do_wave']: - dep_dict = {'type': 'metatask', 'name': f'{self.run}_wavepostsbs'} - deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'task', 'name': f'{self.run}_wavepostpnt'} - deps.append(rocoto.add_dependency(dep_dict)) - if self.options['do_wave_bnd']: - dep_dict = {'type': 'task', 'name': f'{self.run}_wavepostbndpnt'} - deps.append(rocoto.add_dependency(dep_dict)) if self.options['do_ocean']: if self.run in ['gfs']: dep_dict = {'type': 'metatask', 'name': f'{self.run}_ocean_prod'} @@ -2326,15 +2306,6 @@ def arch_vrfy(self): if self.run in ['gfs']: dep_dict = {'type': 'metatask', 'name': f'{self.run}_ice_prod'} deps.append(rocoto.add_dependency(dep_dict)) - # MOS job dependencies - if self.run in ['gfs'] and self.options['do_mos']: - mos_jobs = ["stn_prep", "grd_prep", "ext_stn_prep", "ext_grd_prep", - "stn_fcst", "grd_fcst", "ext_stn_fcst", "ext_grd_fcst", - "stn_prdgen", "grd_prdgen", "ext_stn_prdgen", "ext_grd_prdgen", - "wx_prdgen", "wx_ext_prdgen"] - for job in mos_jobs: - dep_dict = {'type': 'task', 'name': f'{self.run}_mos_{job}'} - deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) From 6db7a6d1baf6f4d608cc66a7f7bee8c391093cc2 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Fri, 24 Jan 2025 11:26:42 -0500 Subject: [PATCH 087/105] modified workflow\rocoto\gfs_tasks.py --- parm/config/gfs/config.earc_vrfy | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/parm/config/gfs/config.earc_vrfy b/parm/config/gfs/config.earc_vrfy index b45b52786f..9fe52b7ee4 100755 --- a/parm/config/gfs/config.earc_vrfy +++ b/parm/config/gfs/config.earc_vrfy @@ -8,26 +8,6 @@ echo "BEGIN: config.earc_vrfy" # Get task specific resources" . "${EXPDIR}/config.resources" "earc_vrfy" -# Set the number of ensemble members to archive per earc_vrfy job -case "${CASE_ENS}" in - "C48" | "C96") - export NMEM_EARCGRP=80 - ;; - "C192") - export NMEM_EARCGRP=20 - ;; - "C384" | "C768") - export NMEM_EARCGRP=10 - ;; - "C1152") - export NMEM_EARCGRP=4 - ;; - *) - echo "FATAL ERROR: Unknown ensemble resolution ${CASE_ENS}, ABORT!" - exit 1 - ;; -esac - #--starting and ending hours of previous cycles to be removed from rotating directory export RMOLDSTD_ENKF=144 export RMOLDEND_ENKF=24 From 80c9821e7eb3062e34034f713de5bc537585ea8f Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Fri, 24 Jan 2025 15:31:52 -0500 Subject: [PATCH 088/105] modified parm\config\gfs\config.resources --- parm/config/gfs/config.resources | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index 8753fa37dd..c966c05b74 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -1059,7 +1059,7 @@ case ${step} in export is_exclusive=True ;; - "arch_tars" | "earc_vrfy" | "earc_tars" | "getic") + "arch_tars" | "earc_tars" | "getic") walltime="06:00:00" ntasks=1 tasks_per_node=1 @@ -1067,7 +1067,7 @@ case ${step} in memory="4096M" ;; - "arch_vrfy") + "arch_vrfy" | "earc_vrfy") walltime="00:15:00" ntasks=1 tasks_per_node=1 From 5594f9dda7ac9906d3de355644a8c334e775d362 Mon Sep 17 00:00:00 2001 From: Anton Fernando Date: Fri, 24 Jan 2025 20:51:00 +0000 Subject: [PATCH 089/105] change permission --- jobs/JGDAS_ENKF_ARCHIVE_VRFY | 0 jobs/JGLOBAL_ARCHIVE_TARS | 0 jobs/JGLOBAL_ARCHIVE_VRFY | 0 jobs/rocoto/arch_tars.sh | 0 jobs/rocoto/arch_vrfy.sh | 0 parm/config/gefs/config.arch_vrfy | 0 scripts/exgdas_enkf_earc_tars.py | 0 scripts/exgdas_enkf_earc_vrfy.py | 0 scripts/exglobal_archive_tars.py | 0 9 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 jobs/JGDAS_ENKF_ARCHIVE_VRFY mode change 100644 => 100755 jobs/JGLOBAL_ARCHIVE_TARS mode change 100644 => 100755 jobs/JGLOBAL_ARCHIVE_VRFY mode change 100644 => 100755 jobs/rocoto/arch_tars.sh mode change 100644 => 100755 jobs/rocoto/arch_vrfy.sh mode change 100755 => 100644 parm/config/gefs/config.arch_vrfy mode change 100644 => 100755 scripts/exgdas_enkf_earc_tars.py mode change 100644 => 100755 scripts/exgdas_enkf_earc_vrfy.py mode change 100644 => 100755 scripts/exglobal_archive_tars.py diff --git a/jobs/JGDAS_ENKF_ARCHIVE_VRFY b/jobs/JGDAS_ENKF_ARCHIVE_VRFY old mode 100644 new mode 100755 diff --git a/jobs/JGLOBAL_ARCHIVE_TARS b/jobs/JGLOBAL_ARCHIVE_TARS old mode 100644 new mode 100755 diff --git a/jobs/JGLOBAL_ARCHIVE_VRFY b/jobs/JGLOBAL_ARCHIVE_VRFY old mode 100644 new mode 100755 diff --git a/jobs/rocoto/arch_tars.sh b/jobs/rocoto/arch_tars.sh old mode 100644 new mode 100755 diff --git a/jobs/rocoto/arch_vrfy.sh b/jobs/rocoto/arch_vrfy.sh old mode 100644 new mode 100755 diff --git a/parm/config/gefs/config.arch_vrfy b/parm/config/gefs/config.arch_vrfy old mode 100755 new mode 100644 diff --git a/scripts/exgdas_enkf_earc_tars.py b/scripts/exgdas_enkf_earc_tars.py old mode 100644 new mode 100755 diff --git a/scripts/exgdas_enkf_earc_vrfy.py b/scripts/exgdas_enkf_earc_vrfy.py old mode 100644 new mode 100755 diff --git a/scripts/exglobal_archive_tars.py b/scripts/exglobal_archive_tars.py old mode 100644 new mode 100755 From b89bd1556adc5233397c064bc3be9792e45ec705 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Fri, 24 Jan 2025 18:57:20 -0500 Subject: [PATCH 090/105] modified jobs\JGDAS_ENKF_ARCHIVE_VRFY --- jobs/JGDAS_ENKF_ARCHIVE_VRFY | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/jobs/JGDAS_ENKF_ARCHIVE_VRFY b/jobs/JGDAS_ENKF_ARCHIVE_VRFY index 0094401987..1ae0a9c327 100755 --- a/jobs/JGDAS_ENKF_ARCHIVE_VRFY +++ b/jobs/JGDAS_ENKF_ARCHIVE_VRFY @@ -11,7 +11,8 @@ YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_TOP MEMDIR="ensstat" YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ COMIN_ATMOS_ANALYSIS_ENSSTAT:COM_ATMOS_ANALYSIS_TMPL \ COMIN_ATMOS_HISTORY_ENSSTAT:COM_ATMOS_HISTORY_TMPL \ - COMIN_SNOW_ANALYSIS_ENSSTAT:COM_SNOW_ANALYSIS_TMPL + COMIN_SNOW_ANALYSIS_ENSSTAT:COM_SNOW_ANALYSIS_TMPL \ + COMOUT_ATMOS_TRACK:COM_ATMOS_TRACK_TMPL ############################################################### # Run archive script From c5529e97b8ce480615b97b4018793b422c0b7301 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Fri, 24 Jan 2025 19:01:59 -0500 Subject: [PATCH 091/105] modified Jscripts --- jobs/JGDAS_ENKF_ARCHIVE_VRFY | 3 +-- jobs/JGLOBAL_ARCHIVE_VRFY | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/jobs/JGDAS_ENKF_ARCHIVE_VRFY b/jobs/JGDAS_ENKF_ARCHIVE_VRFY index 1ae0a9c327..0094401987 100755 --- a/jobs/JGDAS_ENKF_ARCHIVE_VRFY +++ b/jobs/JGDAS_ENKF_ARCHIVE_VRFY @@ -11,8 +11,7 @@ YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_TOP MEMDIR="ensstat" YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ COMIN_ATMOS_ANALYSIS_ENSSTAT:COM_ATMOS_ANALYSIS_TMPL \ COMIN_ATMOS_HISTORY_ENSSTAT:COM_ATMOS_HISTORY_TMPL \ - COMIN_SNOW_ANALYSIS_ENSSTAT:COM_SNOW_ANALYSIS_TMPL \ - COMOUT_ATMOS_TRACK:COM_ATMOS_TRACK_TMPL + COMIN_SNOW_ANALYSIS_ENSSTAT:COM_SNOW_ANALYSIS_TMPL ############################################################### # Run archive script diff --git a/jobs/JGLOBAL_ARCHIVE_VRFY b/jobs/JGLOBAL_ARCHIVE_VRFY index 03525171ba..7a608df66b 100755 --- a/jobs/JGLOBAL_ARCHIVE_VRFY +++ b/jobs/JGLOBAL_ARCHIVE_VRFY @@ -15,6 +15,7 @@ YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ COMIN_CHEM_ANALYSIS:COM_CHEM_ANALYSIS_TMPL \ COMIN_SNOW_ANALYSIS:COM_SNOW_ANALYSIS_TMPL \ COMIN_OBS:COM_OBS_TMPL \ + COMOUT_ATMOS_TRACK:COM_ATMOS_TRACK_TMPL for grid in "0p25" "0p50" "1p00"; do YMD=${PDY} HH=${cyc} GRID=${grid} declare_from_tmpl -rx \ From e0f7047460f3079c586256293aa25cd212a07ce8 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Fri, 24 Jan 2025 19:06:51 -0500 Subject: [PATCH 092/105] modified scripts\exglobal_archive_vrfy.py --- scripts/exglobal_archive_vrfy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/exglobal_archive_vrfy.py b/scripts/exglobal_archive_vrfy.py index ae7e0d7de1..81cc0897b5 100755 --- a/scripts/exglobal_archive_vrfy.py +++ b/scripts/exglobal_archive_vrfy.py @@ -27,7 +27,7 @@ def main(): # Pull out all the configuration keys needed to run the rest of archive steps keys = ['current_cycle', 'RUN', 'PSLOT', 'ROTDIR', 'PARMgfs', 'ARCDIR', 'MODE', 'DO_JEDIATMENS', 'DO_FIT2OBS', 'DO_JEDIATMVAR' - 'DO_JEDISNOWDA', 'DO_AERO_ANL', 'DO_PREP_OBS_AERO'] + 'DO_JEDISNOWDA', 'DO_AERO_ANL', 'DO_PREP_OBS_AERO', 'NET'] archive_dict = AttrDict() for key in keys: From 1fcaf9b045041d7881411372840678d6e8a90ba2 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Fri, 24 Jan 2025 19:19:37 -0500 Subject: [PATCH 093/105] modified scripts\exglobal_archive_vrfy.py --- scripts/exglobal_archive_vrfy.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/exglobal_archive_vrfy.py b/scripts/exglobal_archive_vrfy.py index 81cc0897b5..cf78389f5e 100755 --- a/scripts/exglobal_archive_vrfy.py +++ b/scripts/exglobal_archive_vrfy.py @@ -25,9 +25,10 @@ def main(): logger.info(f"key ({key}) not found in archive.task_config!") # Pull out all the configuration keys needed to run the rest of archive steps - keys = ['current_cycle', 'RUN', 'PSLOT', 'ROTDIR', 'PARMgfs', - 'ARCDIR', 'MODE', 'DO_JEDIATMENS', 'DO_FIT2OBS', 'DO_JEDIATMVAR' - 'DO_JEDISNOWDA', 'DO_AERO_ANL', 'DO_PREP_OBS_AERO', 'NET'] + keys = ['current_cycle', 'RUN', 'PSLOT', 'ROTDIR', 'PARMgfs', 'VFYARC' + 'ARCDIR', 'MODE', 'DO_JEDIATMENS', 'DO_FIT2OBS', 'DO_JEDIATMVAR', + 'DO_JEDISNOWDA', 'DO_AERO_ANL', 'DO_PREP_OBS_AERO', 'NET', 'MODE', + 'FHMAX_HF_GFS', 'FHOUT_GFS', 'FHMAX_FITS', 'FHMAX', 'FHOUT'] archive_dict = AttrDict() for key in keys: From ce5355fb5855c8c050c4c6e56565f1a3ff269b9f Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Fri, 24 Jan 2025 19:53:26 -0500 Subject: [PATCH 094/105] modified scripts\exglobal_archive_vrfy.py --- scripts/exglobal_archive_vrfy.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/exglobal_archive_vrfy.py b/scripts/exglobal_archive_vrfy.py index cf78389f5e..c49cf11b30 100755 --- a/scripts/exglobal_archive_vrfy.py +++ b/scripts/exglobal_archive_vrfy.py @@ -25,10 +25,10 @@ def main(): logger.info(f"key ({key}) not found in archive.task_config!") # Pull out all the configuration keys needed to run the rest of archive steps - keys = ['current_cycle', 'RUN', 'PSLOT', 'ROTDIR', 'PARMgfs', 'VFYARC' + keys = ['current_cycle', 'RUN', 'PSLOT', 'ROTDIR', 'PARMgfs', 'VFYARC', 'ARCDIR', 'MODE', 'DO_JEDIATMENS', 'DO_FIT2OBS', 'DO_JEDIATMVAR', - 'DO_JEDISNOWDA', 'DO_AERO_ANL', 'DO_PREP_OBS_AERO', 'NET', 'MODE', - 'FHMAX_HF_GFS', 'FHOUT_GFS', 'FHMAX_FITS', 'FHMAX', 'FHOUT'] + 'DO_JEDISNOWDA', 'DO_AERO_ANL', 'DO_PREP_OBS_AERO', 'NET', 'MODE', 'FHOUT_GFS', + 'FHMAX_HF_GFS', 'FHOUT_GFS', 'FHMAX_FITS', 'FHMAX', 'FHOUT', 'FHMAX_GFS'] archive_dict = AttrDict() for key in keys: From e470ff8a0141464eed4bccd932b1e2c101cba663 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Mon, 27 Jan 2025 12:09:55 -0500 Subject: [PATCH 095/105] modified jobs\JGLOBAL_ARCHIVE_TARS --- jobs/JGLOBAL_ARCHIVE_TARS | 1 + 1 file changed, 1 insertion(+) diff --git a/jobs/JGLOBAL_ARCHIVE_TARS b/jobs/JGLOBAL_ARCHIVE_TARS index ee166f152b..624f3d83fe 100755 --- a/jobs/JGLOBAL_ARCHIVE_TARS +++ b/jobs/JGLOBAL_ARCHIVE_TARS @@ -2,6 +2,7 @@ source "${HOMEgfs}/ush/preamble.sh" source "${HOMEgfs}/ush/jjob_header.sh" -e "arch_tars" -c "base arch_tars" +source "${USHgfs}/wave_domain_grid.sh" ############################################## # Set variables used in the script From 5cd87304bb220f6fa246a72620f07e4293381c9d Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Mon, 27 Jan 2025 14:59:10 -0500 Subject: [PATCH 096/105] modified scripts\exgdas_enkf_earc_vrfy.py --- scripts/exgdas_enkf_earc_vrfy.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/exgdas_enkf_earc_vrfy.py b/scripts/exgdas_enkf_earc_vrfy.py index 1c951f44cb..b7dd33a89a 100755 --- a/scripts/exgdas_enkf_earc_vrfy.py +++ b/scripts/exgdas_enkf_earc_vrfy.py @@ -18,8 +18,10 @@ def main(): archive = Archive(config) # Pull out all the configuration keys needed to run the rest of archive steps - keys = ['current_cycle', 'RUN', 'PSLOT', 'ROTDIR', 'PARMgfs', - 'ARCDIR', 'MODE', 'DO_JEDIATMENS', 'DO_FIT2OBS'] + keys = ['current_cycle', 'RUN', 'PSLOT', 'ROTDIR', 'PARMgfs', 'VFYARC', + 'ARCDIR', 'MODE', 'DO_JEDIATMENS', 'DO_FIT2OBS', 'DO_JEDIATMVAR', + 'DO_JEDISNOWDA', 'DO_AERO_ANL', 'DO_PREP_OBS_AERO', 'NET', 'MODE', 'FHOUT_GFS', + 'FHMAX_HF_GFS', 'FHOUT_GFS', 'FHMAX_FITS', 'FHMAX', 'FHOUT', 'FHMAX_GFS'] archive_dict = AttrDict() for key in keys: From c7814d97d0d4cdd0d9e3e1e826cabdb16532c27a Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Tue, 28 Jan 2025 14:05:05 -0500 Subject: [PATCH 097/105] modified workflow\applications\gefs.py --- workflow/applications/gefs.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/workflow/applications/gefs.py b/workflow/applications/gefs.py index ad9472cca2..9650912459 100644 --- a/workflow/applications/gefs.py +++ b/workflow/applications/gefs.py @@ -28,7 +28,7 @@ def _get_app_configs(self, run): Returns the config_files that are involved in gefs """ options = self.run_options[run] - configs = ['stage_ic', 'fcst', 'atmos_products', 'arch_vrfy', 'cleanup'] + configs = ['stage_ic', 'fcst', 'atmos_products'] if options['nens'] > 0: configs += ['efcs', 'atmos_ensstat'] @@ -47,12 +47,11 @@ def _get_app_configs(self, run): if options['do_extractvars']: configs += ['extractvars'] - if options['do_extractvars']: - configs += ['extractvars'] - if options['do_archtar']: configs += ['arch_tars'] + configs += ['arch_vrfy', 'cleanup'] + return configs @staticmethod From fb7187c39823f5067960c806a576a9ef7ede0e4a Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Tue, 28 Jan 2025 14:16:37 -0500 Subject: [PATCH 098/105] modify workflow\rocoto\gefs_tasks.py --- workflow/rocoto/gefs_tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index 0b6e3d1b60..4b341d2c1c 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -631,11 +631,11 @@ def arch_tars(self): def cleanup(self): deps = [] + dep_dict = {'type': 'task', 'name': 'gefs_arch_vrfy'} + deps.append(rocoto.add_dependency(dep_dict)) if self.options['do_archtar']: dep_dict = {'type': 'task', 'name': 'gefs_arch_tars'} deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'task', 'name': 'gefs_arch_vrfy'} - deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('cleanup') task_name = 'gefs_cleanup' From b4c3148f8083705569ad3f3612ebb87054835c48 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Tue, 28 Jan 2025 14:19:03 -0500 Subject: [PATCH 099/105] modified workflow\rocoto\gefs_tasks.py --- workflow/rocoto/gefs_tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index 4b341d2c1c..8b1522ae32 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -636,7 +636,7 @@ def cleanup(self): if self.options['do_archtar']: dep_dict = {'type': 'task', 'name': 'gefs_arch_tars'} deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) + dependencies = rocoto.create_dependency(dep=deps, dep_condition='and') resources = self.get_resource('cleanup') task_name = 'gefs_cleanup' task_dict = {'task_name': task_name, From 3db17b876d4bc8f5553eb1cb8f5b608f4d32cddb Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Tue, 28 Jan 2025 15:45:57 -0500 Subject: [PATCH 100/105] modified ARCH jscripts --- jobs/JGLOBAL_ARCHIVE_TARS | 2 +- jobs/JGLOBAL_ARCHIVE_VRFY | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/jobs/JGLOBAL_ARCHIVE_TARS b/jobs/JGLOBAL_ARCHIVE_TARS index 624f3d83fe..e89370bd37 100755 --- a/jobs/JGLOBAL_ARCHIVE_TARS +++ b/jobs/JGLOBAL_ARCHIVE_TARS @@ -1,7 +1,7 @@ #! /usr/bin/env bash source "${HOMEgfs}/ush/preamble.sh" -source "${HOMEgfs}/ush/jjob_header.sh" -e "arch_tars" -c "base arch_tars" +source "${HOMEgfs}/ush/jjob_header.sh" -e "arch_tars" -c "base arch_tars wave" source "${USHgfs}/wave_domain_grid.sh" ############################################## diff --git a/jobs/JGLOBAL_ARCHIVE_VRFY b/jobs/JGLOBAL_ARCHIVE_VRFY index 7a608df66b..01dd876ee1 100755 --- a/jobs/JGLOBAL_ARCHIVE_VRFY +++ b/jobs/JGLOBAL_ARCHIVE_VRFY @@ -1,7 +1,7 @@ #! /usr/bin/env bash source "${HOMEgfs}/ush/preamble.sh" -source "${HOMEgfs}/ush/jjob_header.sh" -e "arch_vrfy" -c "base arch_vrfy" +source "${HOMEgfs}/ush/jjob_header.sh" -e "arch_vrfy" -c "base arch_vrfy wave" ############################################## From 5c50e6a2b9f2fea7b6d1b583bccebe20244acacd Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Tue, 28 Jan 2025 16:00:21 -0500 Subject: [PATCH 101/105] modified scripts\exglobal_archive_vrfy.py --- scripts/exglobal_archive_vrfy.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/exglobal_archive_vrfy.py b/scripts/exglobal_archive_vrfy.py index c49cf11b30..4d08b994f0 100755 --- a/scripts/exglobal_archive_vrfy.py +++ b/scripts/exglobal_archive_vrfy.py @@ -25,8 +25,8 @@ def main(): logger.info(f"key ({key}) not found in archive.task_config!") # Pull out all the configuration keys needed to run the rest of archive steps - keys = ['current_cycle', 'RUN', 'PSLOT', 'ROTDIR', 'PARMgfs', 'VFYARC', - 'ARCDIR', 'MODE', 'DO_JEDIATMENS', 'DO_FIT2OBS', 'DO_JEDIATMVAR', + keys = ['current_cycle', 'RUN', 'PSLOT', 'ROTDIR', 'PARMgfs', 'VFYARC', 'REPLAY_ICS', + 'ARCDIR', 'MODE', 'DO_JEDIATMENS', 'DO_FIT2OBS', 'DO_JEDIATMVAR', 'FHMIN_GFS', 'DO_JEDISNOWDA', 'DO_AERO_ANL', 'DO_PREP_OBS_AERO', 'NET', 'MODE', 'FHOUT_GFS', 'FHMAX_HF_GFS', 'FHOUT_GFS', 'FHMAX_FITS', 'FHMAX', 'FHOUT', 'FHMAX_GFS'] From 62892b595b753bcb8ef087e80fbeadf01ff9fa77 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Wed, 29 Jan 2025 09:58:50 -0500 Subject: [PATCH 102/105] added updated that failed 3DVar job --- parm/archive/gfsa.yaml.j2 | 2 +- workflow/applications/gfs_cycled.py | 5 +++-- workflow/rocoto/gfs_tasks.py | 20 ++++++++++++++++++++ 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/parm/archive/gfsa.yaml.j2 b/parm/archive/gfsa.yaml.j2 index 4efe281120..a7cfaafcd5 100644 --- a/parm/archive/gfsa.yaml.j2 +++ b/parm/archive/gfsa.yaml.j2 @@ -6,7 +6,7 @@ gfsa: # Logs # TODO explicitly name all logs to include {% for log in glob("logs/" ~ cycle_YMDH ~ "/gfs*.log") %} - {% if not "gfs_arch.log" in log %} + {% if not "gfs_arch_tars.log" in log %} - "{{ log }}" {% endif %} {% endfor %} diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index 6962e97263..8c0cc0e9fd 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -301,10 +301,11 @@ def get_task_names(self): 'mos_stn_prdgen', 'mos_grd_prdgen', 'mos_ext_stn_prdgen', 'mos_ext_grd_prdgen', 'mos_wx_prdgen', 'mos_wx_ext_prdgen'] - # Last two items + # Last items + task_names[run] += ['arch_vrfy'] if options['do_archtar']: task_names[run] += ['arch_tars'] - task_names[run] += ['arch_vrfy', 'cleanup'] + task_names[run] += ['cleanup'] # Ensemble tasks elif 'enkf' in run: diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 526fe01c07..4eae7c7ef9 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -2336,6 +2336,14 @@ def arch_vrfy(self): if self.run in ['gfs']: dep_dict = {'type': 'metatask', 'name': f'{self.run}_ice_prod'} deps.append(rocoto.add_dependency(dep_dict)) + if self.options['do_metp'] and self.run in ['gfs']: + deps2 = [] + # taskvalid only handles regular tasks, so just check the first metp job exists + dep_dict = {'type': 'taskvalid', 'name': f'{self.run}_metpg2g1', 'condition': 'not'} + deps2.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'metatask', 'name': f'{self.run}_metp'} + deps2.append(rocoto.add_dependency(dep_dict)) + deps.append(rocoto.create_dependency(dep_condition='or', dep=deps2)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) @@ -2418,6 +2426,18 @@ def arch_tars(self): dep_dict = {'type': 'task', 'name': f'{self.run}_mos_{job}'} deps.append(rocoto.add_dependency(dep_dict)) + if self.options['do_metp'] and self.run in ['gfs']: + deps2 = [] + # taskvalid only handles regular tasks, so just check the first metp job exists + dep_dict = {'type': 'taskvalid', 'name': f'{self.run}_metpg2g1', 'condition': 'not'} + deps2.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'metatask', 'name': f'{self.run}_metp'} + deps2.append(rocoto.add_dependency(dep_dict)) + deps.append(rocoto.create_dependency(dep_condition='or', dep=deps2)) + + dep_dict = {'type': 'task', 'name': f'{self.run}_arch_vrfy'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('arch_tars') From 5da188965c360f8f9073fa4411fc3e8ddabc438f Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Wed, 29 Jan 2025 10:13:38 -0500 Subject: [PATCH 103/105] modified workflow\rocoto\gfs_tasks.py --- workflow/rocoto/gfs_tasks.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 4eae7c7ef9..ffbe76e2de 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -2336,14 +2336,6 @@ def arch_vrfy(self): if self.run in ['gfs']: dep_dict = {'type': 'metatask', 'name': f'{self.run}_ice_prod'} deps.append(rocoto.add_dependency(dep_dict)) - if self.options['do_metp'] and self.run in ['gfs']: - deps2 = [] - # taskvalid only handles regular tasks, so just check the first metp job exists - dep_dict = {'type': 'taskvalid', 'name': f'{self.run}_metpg2g1', 'condition': 'not'} - deps2.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'metatask', 'name': f'{self.run}_metp'} - deps2.append(rocoto.add_dependency(dep_dict)) - deps.append(rocoto.create_dependency(dep_condition='or', dep=deps2)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) From 541286b80ce6857c6af1541d3bbf71ed89e25e4e Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Wed, 29 Jan 2025 13:29:39 -0500 Subject: [PATCH 104/105] modified workflow\rocoto\gfs_tasks.py --- workflow/rocoto/gfs_tasks.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index ffbe76e2de..7bacda50be 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -1897,9 +1897,6 @@ def fit2obs(self): def metp(self): deps = [] - if self.options['do_archtar']: - dep_dict = {'type': 'task', 'name': f'{self.run}_arch_tars'} - deps.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type': 'task', 'name': f'{self.run}_arch_vrfy'} deps.append(rocoto.add_dependency(dep_dict)) if self._base["interval_gfs"] < to_timedelta("24H"): From 8fac576bce071d0464be01d533354c3e9aff7d8a Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Thu, 30 Jan 2025 11:40:22 -0500 Subject: [PATCH 105/105] modified parm\config\gefs\config.resources --- parm/config/gefs/config.resources | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/config/gefs/config.resources b/parm/config/gefs/config.resources index 9c6522a992..8270ea92b5 100644 --- a/parm/config/gefs/config.resources +++ b/parm/config/gefs/config.resources @@ -332,7 +332,7 @@ case ${step} in ;; "arch_vrfy") - export walltime="00:10:00" + export walltime="00:15:00" export ntasks=1 export tasks_per_node=1 export threads_per_task=1