From 32258b087c1022efad02236c101b2bd022a2b96e Mon Sep 17 00:00:00 2001 From: ShannonShields-NOAA <66634320+ShannonShields-NOAA@users.noreply.github.com> Date: Fri, 15 Nov 2024 13:57:54 -0500 Subject: [PATCH 1/6] Feature/subseasonal mpmd fix for stats (#602) * Start MPMD dir changes and add new copy script * Continue MPMD dir changes * Continue making MPMD dir changes for G2G stats * Make MPMD dir changes for G2G stats avg scripts * Make MPMD dir changes for G2G stats anomaly scripts * Start MPMD output dir changes in subseasonal_util.py * Continue MPMD dir changes in subseasonal_util.py for G2G stats * Make MPMD output dir changes in G2G metplus configs * Remove WARNING print statement * Rename copy list * Rename copy list * Set up MPMD output dirs for G2O stats job scripts * Set up MPMD output dirs for G2O stats anomaly scripts * Set up MPMD output dirs for G2O stats anomaly scripts * Start editing G2O avg scripts for MPMD output dir changes * Edit G2O avg scripts for MPMD output dir changes * Edit subseasonal_util.py for MPMD output dir changes * Undo plot changes to test stats only * Edit G2O METplus configs for MPMD output dir changes * Fix issues in subseasonal_util.py * Remove commented out code --------- Co-authored-by: shannon shields Co-authored-by: shannon shields Co-authored-by: shannon shields Co-authored-by: shannon shields Co-authored-by: shannon shields Co-authored-by: shannon shields Co-authored-by: shannon shields Co-authored-by: shannon shields --- .../GenEnsProd_fcstCFS_Days6_10NetCDF.conf | 4 +- .../GenEnsProd_fcstCFS_WeeklyNetCDF.conf | 4 +- .../GenEnsProd_fcstCFS_Weeks3_4NetCDF.conf | 4 +- ...enEnsProd_fcstSUBSEASONAL_DailyNetCDF.conf | 4 +- ...nsProd_fcstSUBSEASONAL_Days6_10NetCDF.conf | 4 +- ...EnsProd_fcstSUBSEASONAL_MonthlyNetCDF.conf | 4 +- ...nEnsProd_fcstSUBSEASONAL_WeeklyNetCDF.conf | 4 +- ...nsProd_fcstSUBSEASONAL_Weeks3_4NetCDF.conf | 4 +- .../GridStat_fcstSUBSEASONAL_DailyNetCDF.conf | 4 +- ...ridStat_fcstSUBSEASONAL_MonthlyNetCDF.conf | 4 +- ...GridStat_fcstSUBSEASONAL_WeeklyNetCDF.conf | 4 +- ...tSUBSEASONAL_obsECMWF_Days6_10AvgAnom.conf | 4 +- ...cstSUBSEASONAL_obsECMWF_WeeklyAvgAnom.conf | 4 +- ...tSUBSEASONAL_obsECMWF_Weeks3_4AvgAnom.conf | 4 +- ...ASONAL_obsECMWF_climoERA5_Days6_10Avg.conf | 4 +- ...NAL_obsECMWF_climoERA5_Days6_10NetCDF.conf | 4 +- ...SEASONAL_obsECMWF_climoERA5_WeeklyAvg.conf | 4 +- ...SONAL_obsECMWF_climoERA5_WeeklyNetCDF.conf | 4 +- ...ASONAL_obsECMWF_climoERA5_Weeks3_4Avg.conf | 4 +- ...NAL_obsECMWF_climoERA5_Weeks3_4NetCDF.conf | 4 +- ...cstSUBSEASONAL_obsGFS_Days6_10AvgAnom.conf | 4 +- ..._fcstSUBSEASONAL_obsGFS_WeeklyAvgAnom.conf | 4 +- ...cstSUBSEASONAL_obsGFS_Weeks3_4AvgAnom.conf | 4 +- ...SEASONAL_obsGFS_climoERA5_Days6_10Avg.conf | 4 +- ...SONAL_obsGFS_climoERA5_Days6_10NetCDF.conf | 4 +- ...UBSEASONAL_obsGFS_climoERA5_WeeklyAvg.conf | 4 +- ...EASONAL_obsGFS_climoERA5_WeeklyNetCDF.conf | 4 +- ...SEASONAL_obsGFS_climoERA5_Weeks3_4Avg.conf | 4 +- ...SONAL_obsGFS_climoERA5_Weeks3_4NetCDF.conf | 4 +- ...stSUBSEASONAL_obsGHRSST_OSPO_DailyAvg.conf | 4 +- ...SUBSEASONAL_obsGHRSST_OSPO_MonthlyAvg.conf | 4 +- ...tSUBSEASONAL_obsGHRSST_OSPO_WeeklyAvg.conf | 4 +- ...fcstSUBSEASONAL_obsOSI-SAF_MonthlyAvg.conf | 4 +- ..._fcstSUBSEASONAL_obsOSI-SAF_WeeklyAvg.conf | 4 +- .../StatAnalysis_fcstSUBSEASONAL.conf | 4 +- ...nsProd_fcstSUBSEASONAL_Days6_10NetCDF.conf | 4 +- ...nEnsProd_fcstSUBSEASONAL_WeeklyNetCDF.conf | 4 +- ...nsProd_fcstSUBSEASONAL_Weeks3_4NetCDF.conf | 4 +- .../grid2obs/PB2NC_obsPrepbufr_Weeks3_4.conf | 4 +- ...NAL_obsPrepbufr_climoERA5_Days6_10MPR.conf | 5 +- ...SONAL_obsPrepbufr_climoERA5_WeeklyMPR.conf | 5 +- ...NAL_obsPrepbufr_climoERA5_Weeks3_4MPR.conf | 5 +- .../StatAnalysis_fcstSUBSEASONAL.conf | 4 +- ...ASONAL_obsPrepbufr_Days6_10MPRtoSL1L2.conf | 8 +- ...SEASONAL_obsPrepbufr_WeeklyMPRtoSL1L2.conf | 8 +- ...ASONAL_obsPrepbufr_Weeks3_4MPRtoSL1L2.conf | 8 +- ...repbufr_climoERA5_Days6_10MPRtoSAL1L2.conf | 8 +- ...sPrepbufr_climoERA5_WeeklyMPRtoSAL1L2.conf | 8 +- ...repbufr_climoERA5_Weeks3_4MPRtoSAL1L2.conf | 8 +- .../exevs_subseasonal_grid2grid_stats.sh | 2 + .../exevs_subseasonal_grid2obs_stats.sh | 2 + .../create_METplus_subseasonal_output_dirs.py | 7 +- .../subseasonal_copy_job_dir_output.py | 86 +++ ...asonal_stats_grid2grid_create_daily_avg.py | 19 +- ...2grid_create_daily_reformat_job_scripts.py | 14 +- ...stats_grid2grid_create_days6_10_anomaly.py | 76 +- ...nal_stats_grid2grid_create_days6_10_avg.py | 19 +- ...id_create_days6_10_reformat_job_scripts.py | 58 +- ...onal_stats_grid2grid_create_job_scripts.py | 26 +- ...onal_stats_grid2grid_create_monthly_avg.py | 19 +- ...rid_create_monthly_reformat_job_scripts.py | 14 +- ...2grid_create_week4_reformat_job_scripts.py | 17 +- ...l_stats_grid2grid_create_weekly_anomaly.py | 76 +- ...sonal_stats_grid2grid_create_weekly_avg.py | 19 +- ...grid_create_weekly_reformat_job_scripts.py | 58 +- ...stats_grid2grid_create_weeks3_4_anomaly.py | 76 +- ...nal_stats_grid2grid_create_weeks3_4_avg.py | 19 +- ...id_create_weeks3_4_reformat_job_scripts.py | 58 +- ..._stats_grid2obs_create_days6_10_anomaly.py | 126 ++-- ...bs_create_days6_10_assemble_job_scripts.py | 37 +- ...onal_stats_grid2obs_create_days6_10_avg.py | 71 +- ...bs_create_days6_10_reformat_job_scripts.py | 20 +- ...sonal_stats_grid2obs_create_job_scripts.py | 54 +- ...al_stats_grid2obs_create_weekly_anomaly.py | 126 ++-- ...2obs_create_weekly_assemble_job_scripts.py | 37 +- ...asonal_stats_grid2obs_create_weekly_avg.py | 71 +- ...2obs_create_weekly_reformat_job_scripts.py | 20 +- ..._stats_grid2obs_create_weeks3_4_anomaly.py | 126 ++-- ...bs_create_weeks3_4_assemble_job_scripts.py | 37 +- ...onal_stats_grid2obs_create_weeks3_4_avg.py | 71 +- ...bs_create_weeks3_4_reformat_job_scripts.py | 38 +- ush/subseasonal/subseasonal_util.py | 692 ++++++------------ 82 files changed, 1218 insertions(+), 1196 deletions(-) create mode 100644 ush/subseasonal/subseasonal_copy_job_dir_output.py diff --git a/parm/metplus_config/stats/subseasonal/grid2grid/GenEnsProd_fcstCFS_Days6_10NetCDF.conf b/parm/metplus_config/stats/subseasonal/grid2grid/GenEnsProd_fcstCFS_Days6_10NetCDF.conf index 573464bd3..443cbeee0 100644 --- a/parm/metplus_config/stats/subseasonal/grid2grid/GenEnsProd_fcstCFS_Days6_10NetCDF.conf +++ b/parm/metplus_config/stats/subseasonal/grid2grid/GenEnsProd_fcstCFS_Days6_10NetCDF.conf @@ -9,9 +9,9 @@ SCRUB_STAGING_DIR = False MODEL = {ENV[MODEL]} OBTYPE = {ENV[MODEL]} #### Output -OUTPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}_{ENV[STEP]}/METplus_output +OUTPUT_BASE = {ENV[job_num_work_dir]} LOG_DIR = {OUTPUT_BASE}/logs -TMP_DIR = {OUTPUT_BASE}/tmp +TMP_DIR = {ENV[MET_TMP_DIR]} STAGING_DIR = {OUTPUT_BASE}/stage GEN_ENS_PROD_OUTPUT_DIR = {OUTPUT_BASE} GEN_ENS_PROD_OUTPUT_TEMPLATE = {ENV[RUN]}.{ENV[DATE]}/{MODEL}/{ENV[VERIF_CASE]}/gen_ens_prod_{ENV[VERIF_TYPE]}_{ENV[job_name]}_FHR{lead?fmt=%3H}_{valid?fmt=%Y%m%d}_{valid?fmt=%H}0000V_ens.nc diff --git a/parm/metplus_config/stats/subseasonal/grid2grid/GenEnsProd_fcstCFS_WeeklyNetCDF.conf b/parm/metplus_config/stats/subseasonal/grid2grid/GenEnsProd_fcstCFS_WeeklyNetCDF.conf index 6f8963f6c..89987434d 100644 --- a/parm/metplus_config/stats/subseasonal/grid2grid/GenEnsProd_fcstCFS_WeeklyNetCDF.conf +++ b/parm/metplus_config/stats/subseasonal/grid2grid/GenEnsProd_fcstCFS_WeeklyNetCDF.conf @@ -9,9 +9,9 @@ SCRUB_STAGING_DIR = False MODEL = {ENV[MODEL]} OBTYPE = {ENV[MODEL]} #### Output -OUTPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}_{ENV[STEP]}/METplus_output +OUTPUT_BASE = {ENV[job_num_work_dir]} LOG_DIR = {OUTPUT_BASE}/logs -TMP_DIR = {OUTPUT_BASE}/tmp +TMP_DIR = {ENV[MET_TMP_DIR]} STAGING_DIR = {OUTPUT_BASE}/stage GEN_ENS_PROD_OUTPUT_DIR = {OUTPUT_BASE} GEN_ENS_PROD_OUTPUT_TEMPLATE = {ENV[RUN]}.{ENV[DATE]}/{MODEL}/{ENV[VERIF_CASE]}/gen_ens_prod_{ENV[VERIF_TYPE]}_{ENV[job_name]}_FHR{lead?fmt=%3H}_{valid?fmt=%Y%m%d}_{valid?fmt=%H}0000V_ens.nc diff --git a/parm/metplus_config/stats/subseasonal/grid2grid/GenEnsProd_fcstCFS_Weeks3_4NetCDF.conf b/parm/metplus_config/stats/subseasonal/grid2grid/GenEnsProd_fcstCFS_Weeks3_4NetCDF.conf index 0df1c250b..49352e6de 100644 --- a/parm/metplus_config/stats/subseasonal/grid2grid/GenEnsProd_fcstCFS_Weeks3_4NetCDF.conf +++ b/parm/metplus_config/stats/subseasonal/grid2grid/GenEnsProd_fcstCFS_Weeks3_4NetCDF.conf @@ -9,9 +9,9 @@ SCRUB_STAGING_DIR = False MODEL = {ENV[MODEL]} OBTYPE = {ENV[MODEL]} #### Output -OUTPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}_{ENV[STEP]}/METplus_output +OUTPUT_BASE = {ENV[job_num_work_dir]} LOG_DIR = {OUTPUT_BASE}/logs -TMP_DIR = {OUTPUT_BASE}/tmp +TMP_DIR = {ENV[MET_TMP_DIR]} STAGING_DIR = {OUTPUT_BASE}/stage GEN_ENS_PROD_OUTPUT_DIR = {OUTPUT_BASE} GEN_ENS_PROD_OUTPUT_TEMPLATE = {ENV[RUN]}.{ENV[DATE]}/{MODEL}/{ENV[VERIF_CASE]}/gen_ens_prod_{ENV[VERIF_TYPE]}_{ENV[job_name]}_FHR{lead?fmt=%3H}_{valid?fmt=%Y%m%d}_{valid?fmt=%H}0000V_ens.nc diff --git a/parm/metplus_config/stats/subseasonal/grid2grid/GenEnsProd_fcstSUBSEASONAL_DailyNetCDF.conf b/parm/metplus_config/stats/subseasonal/grid2grid/GenEnsProd_fcstSUBSEASONAL_DailyNetCDF.conf index b35e97fff..1ae1bc88e 100644 --- a/parm/metplus_config/stats/subseasonal/grid2grid/GenEnsProd_fcstSUBSEASONAL_DailyNetCDF.conf +++ b/parm/metplus_config/stats/subseasonal/grid2grid/GenEnsProd_fcstSUBSEASONAL_DailyNetCDF.conf @@ -9,9 +9,9 @@ SCRUB_STAGING_DIR = False MODEL = {ENV[MODEL]} OBTYPE = {ENV[MODEL]} #### Output -OUTPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}_{ENV[STEP]}/METplus_output +OUTPUT_BASE = {ENV[job_num_work_dir]} LOG_DIR = {OUTPUT_BASE}/logs -TMP_DIR = {OUTPUT_BASE}/tmp +TMP_DIR = {ENV[MET_TMP_DIR]} STAGING_DIR = {OUTPUT_BASE}/stage GEN_ENS_PROD_OUTPUT_DIR = {OUTPUT_BASE} GEN_ENS_PROD_OUTPUT_TEMPLATE = {ENV[RUN]}.{ENV[DATE]}/{MODEL}/{ENV[VERIF_CASE]}/gen_ens_prod_{ENV[VERIF_TYPE]}_{ENV[job_name]}_FHR{lead?fmt=%3H}_{valid?fmt=%Y%m%d}_{valid?fmt=%H}0000V_ens.nc diff --git a/parm/metplus_config/stats/subseasonal/grid2grid/GenEnsProd_fcstSUBSEASONAL_Days6_10NetCDF.conf b/parm/metplus_config/stats/subseasonal/grid2grid/GenEnsProd_fcstSUBSEASONAL_Days6_10NetCDF.conf index 6a4172b10..1c17d9b11 100644 --- a/parm/metplus_config/stats/subseasonal/grid2grid/GenEnsProd_fcstSUBSEASONAL_Days6_10NetCDF.conf +++ b/parm/metplus_config/stats/subseasonal/grid2grid/GenEnsProd_fcstSUBSEASONAL_Days6_10NetCDF.conf @@ -9,9 +9,9 @@ SCRUB_STAGING_DIR = False MODEL = {ENV[MODEL]} OBTYPE = {ENV[MODEL]} #### Output -OUTPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}_{ENV[STEP]}/METplus_output +OUTPUT_BASE = {ENV[job_num_work_dir]} LOG_DIR = {OUTPUT_BASE}/logs -TMP_DIR = {OUTPUT_BASE}/tmp +TMP_DIR = {ENV[MET_TMP_DIR]} STAGING_DIR = {OUTPUT_BASE}/stage GEN_ENS_PROD_OUTPUT_DIR = {OUTPUT_BASE} GEN_ENS_PROD_OUTPUT_TEMPLATE = {ENV[RUN]}.{ENV[DATE]}/{MODEL}/{ENV[VERIF_CASE]}/gen_ens_prod_{ENV[VERIF_TYPE]}_{ENV[job_name]}_FHR{lead?fmt=%3H}_{valid?fmt=%Y%m%d}_{valid?fmt=%H}0000V_ens.nc diff --git a/parm/metplus_config/stats/subseasonal/grid2grid/GenEnsProd_fcstSUBSEASONAL_MonthlyNetCDF.conf b/parm/metplus_config/stats/subseasonal/grid2grid/GenEnsProd_fcstSUBSEASONAL_MonthlyNetCDF.conf index bc550a37f..480edab56 100644 --- a/parm/metplus_config/stats/subseasonal/grid2grid/GenEnsProd_fcstSUBSEASONAL_MonthlyNetCDF.conf +++ b/parm/metplus_config/stats/subseasonal/grid2grid/GenEnsProd_fcstSUBSEASONAL_MonthlyNetCDF.conf @@ -9,9 +9,9 @@ SCRUB_STAGING_DIR = False MODEL = {ENV[MODEL]} OBTYPE = {ENV[MODEL]} #### Output -OUTPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}_{ENV[STEP]}/METplus_output +OUTPUT_BASE = {ENV[job_num_work_dir]} LOG_DIR = {OUTPUT_BASE}/logs -TMP_DIR = {OUTPUT_BASE}/tmp +TMP_DIR = {ENV[MET_TMP_DIR]} STAGING_DIR = {OUTPUT_BASE}/stage GEN_ENS_PROD_OUTPUT_DIR = {OUTPUT_BASE} GEN_ENS_PROD_OUTPUT_TEMPLATE = {ENV[RUN]}.{ENV[DATE]}/{MODEL}/{ENV[VERIF_CASE]}/gen_ens_prod_{ENV[VERIF_TYPE]}_{ENV[job_name]}_FHR{lead?fmt=%3H}_{valid?fmt=%Y%m%d}_{valid?fmt=%H}0000V_ens.nc diff --git a/parm/metplus_config/stats/subseasonal/grid2grid/GenEnsProd_fcstSUBSEASONAL_WeeklyNetCDF.conf b/parm/metplus_config/stats/subseasonal/grid2grid/GenEnsProd_fcstSUBSEASONAL_WeeklyNetCDF.conf index d4474f52f..b01236c84 100644 --- a/parm/metplus_config/stats/subseasonal/grid2grid/GenEnsProd_fcstSUBSEASONAL_WeeklyNetCDF.conf +++ b/parm/metplus_config/stats/subseasonal/grid2grid/GenEnsProd_fcstSUBSEASONAL_WeeklyNetCDF.conf @@ -9,9 +9,9 @@ SCRUB_STAGING_DIR = False MODEL = {ENV[MODEL]} OBTYPE = {ENV[MODEL]} #### Output -OUTPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}_{ENV[STEP]}/METplus_output +OUTPUT_BASE = {ENV[job_num_work_dir]} LOG_DIR = {OUTPUT_BASE}/logs -TMP_DIR = {OUTPUT_BASE}/tmp +TMP_DIR = {ENV[MET_TMP_DIR]} STAGING_DIR = {OUTPUT_BASE}/stage GEN_ENS_PROD_OUTPUT_DIR = {OUTPUT_BASE} GEN_ENS_PROD_OUTPUT_TEMPLATE = {ENV[RUN]}.{ENV[DATE]}/{MODEL}/{ENV[VERIF_CASE]}/gen_ens_prod_{ENV[VERIF_TYPE]}_{ENV[job_name]}_FHR{lead?fmt=%3H}_{valid?fmt=%Y%m%d}_{valid?fmt=%H}0000V_ens.nc diff --git a/parm/metplus_config/stats/subseasonal/grid2grid/GenEnsProd_fcstSUBSEASONAL_Weeks3_4NetCDF.conf b/parm/metplus_config/stats/subseasonal/grid2grid/GenEnsProd_fcstSUBSEASONAL_Weeks3_4NetCDF.conf index e82c59bde..d1b43a217 100644 --- a/parm/metplus_config/stats/subseasonal/grid2grid/GenEnsProd_fcstSUBSEASONAL_Weeks3_4NetCDF.conf +++ b/parm/metplus_config/stats/subseasonal/grid2grid/GenEnsProd_fcstSUBSEASONAL_Weeks3_4NetCDF.conf @@ -9,9 +9,9 @@ SCRUB_STAGING_DIR = False MODEL = {ENV[MODEL]} OBTYPE = {ENV[MODEL]} #### Output -OUTPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}_{ENV[STEP]}/METplus_output +OUTPUT_BASE = {ENV[job_num_work_dir]} LOG_DIR = {OUTPUT_BASE}/logs -TMP_DIR = {OUTPUT_BASE}/tmp +TMP_DIR = {ENV[MET_TMP_DIR]} STAGING_DIR = {OUTPUT_BASE}/stage GEN_ENS_PROD_OUTPUT_DIR = {OUTPUT_BASE} GEN_ENS_PROD_OUTPUT_TEMPLATE = {ENV[RUN]}.{ENV[DATE]}/{MODEL}/{ENV[VERIF_CASE]}/gen_ens_prod_{ENV[VERIF_TYPE]}_{ENV[job_name]}_FHR{lead?fmt=%3H}_{valid?fmt=%Y%m%d}_{valid?fmt=%H}0000V_ens.nc diff --git a/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_DailyNetCDF.conf b/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_DailyNetCDF.conf index 94d8d90db..54ccb9f6c 100644 --- a/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_DailyNetCDF.conf +++ b/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_DailyNetCDF.conf @@ -10,9 +10,9 @@ OBS_GRID_STAT_INPUT_DATATYPE = NETCDF OBS_IS_PROB = False SCRUB_STAGING_DIR = False #### Output -OUTPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}_{ENV[STEP]}/METplus_output +OUTPUT_BASE = {ENV[job_num_work_dir]} LOG_DIR = {OUTPUT_BASE}/logs -TMP_DIR = {OUTPUT_BASE}/tmp +TMP_DIR = {ENV[MET_TMP_DIR]} STAGING_DIR = {OUTPUT_BASE}/stage GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} GRID_STAT_OUTPUT_TEMPLATE = {ENV[RUN]}.{ENV[DATE]}/{MODEL}/{ENV[VERIF_CASE]} diff --git a/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_MonthlyNetCDF.conf b/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_MonthlyNetCDF.conf index 4507ebedf..c09097490 100644 --- a/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_MonthlyNetCDF.conf +++ b/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_MonthlyNetCDF.conf @@ -10,9 +10,9 @@ OBS_GRID_STAT_INPUT_DATATYPE = NETCDF OBS_IS_PROB = False SCRUB_STAGING_DIR = False #### Output -OUTPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}_{ENV[STEP]}/METplus_output +OUTPUT_BASE = {ENV[job_num_work_dir]} LOG_DIR = {OUTPUT_BASE}/logs -TMP_DIR = {OUTPUT_BASE}/tmp +TMP_DIR = {ENV[MET_TMP_DIR]} STAGING_DIR = {OUTPUT_BASE}/stage GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} GRID_STAT_OUTPUT_TEMPLATE = {ENV[RUN]}.{ENV[DATE]}/{MODEL}/{ENV[VERIF_CASE]} diff --git a/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_WeeklyNetCDF.conf b/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_WeeklyNetCDF.conf index da2f0a662..579105cc5 100644 --- a/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_WeeklyNetCDF.conf +++ b/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_WeeklyNetCDF.conf @@ -10,9 +10,9 @@ OBS_GRID_STAT_INPUT_DATATYPE = NETCDF OBS_IS_PROB = False SCRUB_STAGING_DIR = False #### Output -OUTPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}_{ENV[STEP]}/METplus_output +OUTPUT_BASE = {ENV[job_num_work_dir]} LOG_DIR = {OUTPUT_BASE}/logs -TMP_DIR = {OUTPUT_BASE}/tmp +TMP_DIR = {ENV[MET_TMP_DIR]} STAGING_DIR = {OUTPUT_BASE}/stage GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} GRID_STAT_OUTPUT_TEMPLATE = {ENV[RUN]}.{ENV[DATE]}/{MODEL}/{ENV[VERIF_CASE]} diff --git a/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsECMWF_Days6_10AvgAnom.conf b/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsECMWF_Days6_10AvgAnom.conf index 2d7b9bdf4..c0b92ee89 100644 --- a/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsECMWF_Days6_10AvgAnom.conf +++ b/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsECMWF_Days6_10AvgAnom.conf @@ -11,9 +11,9 @@ OBS_GRID_STAT_INPUT_DATATYPE = NETCDF OBS_IS_PROB = False SCRUB_STAGING_DIR = False #### Output -OUTPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}_{ENV[STEP]}/METplus_output +OUTPUT_BASE = {ENV[job_num_work_dir]} LOG_DIR = {OUTPUT_BASE}/logs -TMP_DIR = {OUTPUT_BASE}/tmp +TMP_DIR = {ENV[MET_TMP_DIR]} STAGING_DIR = {OUTPUT_BASE}/stage GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} GRID_STAT_OUTPUT_TEMPLATE = {ENV[RUN]}.{ENV[DATE]}/{MODEL}/{ENV[VERIF_CASE]} diff --git a/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsECMWF_WeeklyAvgAnom.conf b/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsECMWF_WeeklyAvgAnom.conf index a90f3c9d0..386596129 100644 --- a/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsECMWF_WeeklyAvgAnom.conf +++ b/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsECMWF_WeeklyAvgAnom.conf @@ -11,9 +11,9 @@ OBS_GRID_STAT_INPUT_DATATYPE = NETCDF OBS_IS_PROB = False SCRUB_STAGING_DIR = False #### Output -OUTPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}_{ENV[STEP]}/METplus_output +OUTPUT_BASE = {ENV[job_num_work_dir]} LOG_DIR = {OUTPUT_BASE}/logs -TMP_DIR = {OUTPUT_BASE}/tmp +TMP_DIR = {ENV[MET_TMP_DIR]} STAGING_DIR = {OUTPUT_BASE}/stage GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} GRID_STAT_OUTPUT_TEMPLATE = {ENV[RUN]}.{ENV[DATE]}/{MODEL}/{ENV[VERIF_CASE]} diff --git a/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsECMWF_Weeks3_4AvgAnom.conf b/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsECMWF_Weeks3_4AvgAnom.conf index de634d1ee..daa058018 100644 --- a/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsECMWF_Weeks3_4AvgAnom.conf +++ b/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsECMWF_Weeks3_4AvgAnom.conf @@ -11,9 +11,9 @@ OBS_GRID_STAT_INPUT_DATATYPE = NETCDF OBS_IS_PROB = False SCRUB_STAGING_DIR = False #### Output -OUTPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}_{ENV[STEP]}/METplus_output +OUTPUT_BASE = {ENV[job_num_work_dir]} LOG_DIR = {OUTPUT_BASE}/logs -TMP_DIR = {OUTPUT_BASE}/tmp +TMP_DIR = {ENV[MET_TMP_DIR]} STAGING_DIR = {OUTPUT_BASE}/stage GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} GRID_STAT_OUTPUT_TEMPLATE = {ENV[RUN]}.{ENV[DATE]}/{MODEL}/{ENV[VERIF_CASE]} diff --git a/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsECMWF_climoERA5_Days6_10Avg.conf b/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsECMWF_climoERA5_Days6_10Avg.conf index 8e6aa04af..45008620f 100644 --- a/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsECMWF_climoERA5_Days6_10Avg.conf +++ b/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsECMWF_climoERA5_Days6_10Avg.conf @@ -12,9 +12,9 @@ OBS_IS_PROB = False SCRUB_STAGING_DIR = False GRID_STAT_CLIMO_MEAN_FILE_NAME = {INPUT_BASE}/METplus_output/{ENV[RUN]}.{ENV[DATE]}/{MODEL}/{ENV[VERIF_CASE]}/days6_10_avg_{ENV[VERIF_TYPE]}_{ENV[job_name]}_init{init?fmt=%Y%m%d%H}_valid{valid?fmt=%Y%m%d%H?shift=-120H}to{valid?fmt=%Y%m%d%H}.nc #### Output -OUTPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}_{ENV[STEP]}/METplus_output +OUTPUT_BASE = {ENV[job_num_work_dir]} LOG_DIR = {OUTPUT_BASE}/logs -TMP_DIR = {OUTPUT_BASE}/tmp +TMP_DIR = {ENV[MET_TMP_DIR]} STAGING_DIR = {OUTPUT_BASE}/stage GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} GRID_STAT_OUTPUT_TEMPLATE = {ENV[RUN]}.{ENV[DATE]}/{MODEL}/{ENV[VERIF_CASE]} diff --git a/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsECMWF_climoERA5_Days6_10NetCDF.conf b/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsECMWF_climoERA5_Days6_10NetCDF.conf index fa289daee..588c529be 100644 --- a/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsECMWF_climoERA5_Days6_10NetCDF.conf +++ b/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsECMWF_climoERA5_Days6_10NetCDF.conf @@ -13,9 +13,9 @@ SCRUB_STAGING_DIR = False GRID_STAT_CLIMO_MEAN_FILE_NAME = {ENV[FIXevs]}/climos/{ENV[RUN]}/era5/mean_{valid?fmt=%m%d} GRID_STAT_CLIMO_STDEV_FILE_NAME = {ENV[FIXevs]}/climos/{ENV[RUN]}/era5/stdev_{valid?fmt=%m%d} #### Output -OUTPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}_{ENV[STEP]}/METplus_output +OUTPUT_BASE = {ENV[job_num_work_dir]} LOG_DIR = {OUTPUT_BASE}/logs -TMP_DIR = {OUTPUT_BASE}/tmp +TMP_DIR = {ENV[MET_TMP_DIR]} STAGING_DIR = {OUTPUT_BASE}/stage GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} GRID_STAT_OUTPUT_TEMPLATE = {ENV[RUN]}.{ENV[DATE]}/{MODEL}/{ENV[VERIF_CASE]} diff --git a/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsECMWF_climoERA5_WeeklyAvg.conf b/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsECMWF_climoERA5_WeeklyAvg.conf index a36ce79d3..083b72ed2 100644 --- a/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsECMWF_climoERA5_WeeklyAvg.conf +++ b/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsECMWF_climoERA5_WeeklyAvg.conf @@ -12,9 +12,9 @@ OBS_IS_PROB = False SCRUB_STAGING_DIR = False GRID_STAT_CLIMO_MEAN_FILE_NAME = {INPUT_BASE}/METplus_output/{ENV[RUN]}.{ENV[DATE]}/{MODEL}/{ENV[VERIF_CASE]}/weekly_avg_{ENV[VERIF_TYPE]}_{ENV[job_name]}_init{init?fmt=%Y%m%d%H}_valid{valid?fmt=%Y%m%d%H?shift=-168H}to{valid?fmt=%Y%m%d%H}.nc #### Output -OUTPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}_{ENV[STEP]}/METplus_output +OUTPUT_BASE = {ENV[job_num_work_dir]} LOG_DIR = {OUTPUT_BASE}/logs -TMP_DIR = {OUTPUT_BASE}/tmp +TMP_DIR = {ENV[MET_TMP_DIR]} STAGING_DIR = {OUTPUT_BASE}/stage GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} GRID_STAT_OUTPUT_TEMPLATE = {ENV[RUN]}.{ENV[DATE]}/{MODEL}/{ENV[VERIF_CASE]} diff --git a/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsECMWF_climoERA5_WeeklyNetCDF.conf b/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsECMWF_climoERA5_WeeklyNetCDF.conf index f81d6bc74..ec6423423 100644 --- a/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsECMWF_climoERA5_WeeklyNetCDF.conf +++ b/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsECMWF_climoERA5_WeeklyNetCDF.conf @@ -13,9 +13,9 @@ SCRUB_STAGING_DIR = False GRID_STAT_CLIMO_MEAN_FILE_NAME = {ENV[FIXevs]}/climos/{ENV[RUN]}/era5/mean_{valid?fmt=%m%d} GRID_STAT_CLIMO_STDEV_FILE_NAME = {ENV[FIXevs]}/climos/{ENV[RUN]}/era5/stdev_{valid?fmt=%m%d} #### Output -OUTPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}_{ENV[STEP]}/METplus_output +OUTPUT_BASE = {ENV[job_num_work_dir]} LOG_DIR = {OUTPUT_BASE}/logs -TMP_DIR = {OUTPUT_BASE}/tmp +TMP_DIR = {ENV[MET_TMP_DIR]} STAGING_DIR = {OUTPUT_BASE}/stage GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} GRID_STAT_OUTPUT_TEMPLATE = {ENV[RUN]}.{ENV[DATE]}/{MODEL}/{ENV[VERIF_CASE]} diff --git a/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsECMWF_climoERA5_Weeks3_4Avg.conf b/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsECMWF_climoERA5_Weeks3_4Avg.conf index 8d74fbed9..5952486b7 100644 --- a/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsECMWF_climoERA5_Weeks3_4Avg.conf +++ b/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsECMWF_climoERA5_Weeks3_4Avg.conf @@ -12,9 +12,9 @@ OBS_IS_PROB = False SCRUB_STAGING_DIR = False GRID_STAT_CLIMO_MEAN_FILE_NAME = {INPUT_BASE}/METplus_output/{ENV[RUN]}.{ENV[DATE]}/{MODEL}/{ENV[VERIF_CASE]}/weeks3_4_avg_{ENV[VERIF_TYPE]}_{ENV[job_name]}_init{init?fmt=%Y%m%d%H}_valid{valid?fmt=%Y%m%d%H?shift=-336H}to{valid?fmt=%Y%m%d%H}.nc #### Output -OUTPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}_{ENV[STEP]}/METplus_output +OUTPUT_BASE = {ENV[job_num_work_dir]} LOG_DIR = {OUTPUT_BASE}/logs -TMP_DIR = {OUTPUT_BASE}/tmp +TMP_DIR = {ENV[MET_TMP_DIR]} STAGING_DIR = {OUTPUT_BASE}/stage GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} GRID_STAT_OUTPUT_TEMPLATE = {ENV[RUN]}.{ENV[DATE]}/{MODEL}/{ENV[VERIF_CASE]} diff --git a/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsECMWF_climoERA5_Weeks3_4NetCDF.conf b/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsECMWF_climoERA5_Weeks3_4NetCDF.conf index 873e40280..e34d7b3a3 100644 --- a/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsECMWF_climoERA5_Weeks3_4NetCDF.conf +++ b/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsECMWF_climoERA5_Weeks3_4NetCDF.conf @@ -13,9 +13,9 @@ SCRUB_STAGING_DIR = False GRID_STAT_CLIMO_MEAN_FILE_NAME = {ENV[FIXevs]}/climos/{ENV[RUN]}/era5/mean_{valid?fmt=%m%d} GRID_STAT_CLIMO_STDEV_FILE_NAME = {ENV[FIXevs]}/climos/{ENV[RUN]}/era5/stdev_{valid?fmt=%m%d} #### Output -OUTPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}_{ENV[STEP]}/METplus_output +OUTPUT_BASE = {ENV[job_num_work_dir]} LOG_DIR = {OUTPUT_BASE}/logs -TMP_DIR = {OUTPUT_BASE}/tmp +TMP_DIR = {ENV[MET_TMP_DIR]} STAGING_DIR = {OUTPUT_BASE}/stage GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} GRID_STAT_OUTPUT_TEMPLATE = {ENV[RUN]}.{ENV[DATE]}/{MODEL}/{ENV[VERIF_CASE]} diff --git a/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsGFS_Days6_10AvgAnom.conf b/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsGFS_Days6_10AvgAnom.conf index d38c60abe..7f9636824 100644 --- a/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsGFS_Days6_10AvgAnom.conf +++ b/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsGFS_Days6_10AvgAnom.conf @@ -11,9 +11,9 @@ OBS_GRID_STAT_INPUT_DATATYPE = NETCDF OBS_IS_PROB = False SCRUB_STAGING_DIR = False #### Output -OUTPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}_{ENV[STEP]}/METplus_output +OUTPUT_BASE = {ENV[job_num_work_dir]} LOG_DIR = {OUTPUT_BASE}/logs -TMP_DIR = {OUTPUT_BASE}/tmp +TMP_DIR = {ENV[MET_TMP_DIR]} STAGING_DIR = {OUTPUT_BASE}/stage GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} GRID_STAT_OUTPUT_TEMPLATE = {ENV[RUN]}.{ENV[DATE]}/{MODEL}/{ENV[VERIF_CASE]} diff --git a/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsGFS_WeeklyAvgAnom.conf b/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsGFS_WeeklyAvgAnom.conf index 5945a4373..f734d2672 100644 --- a/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsGFS_WeeklyAvgAnom.conf +++ b/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsGFS_WeeklyAvgAnom.conf @@ -11,9 +11,9 @@ OBS_GRID_STAT_INPUT_DATATYPE = NETCDF OBS_IS_PROB = False SCRUB_STAGING_DIR = False #### Output -OUTPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}_{ENV[STEP]}/METplus_output +OUTPUT_BASE = {ENV[job_num_work_dir]} LOG_DIR = {OUTPUT_BASE}/logs -TMP_DIR = {OUTPUT_BASE}/tmp +TMP_DIR = {ENV[MET_TMP_DIR]} STAGING_DIR = {OUTPUT_BASE}/stage GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} GRID_STAT_OUTPUT_TEMPLATE = {ENV[RUN]}.{ENV[DATE]}/{MODEL}/{ENV[VERIF_CASE]} diff --git a/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsGFS_Weeks3_4AvgAnom.conf b/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsGFS_Weeks3_4AvgAnom.conf index 8dff07f83..237a67142 100644 --- a/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsGFS_Weeks3_4AvgAnom.conf +++ b/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsGFS_Weeks3_4AvgAnom.conf @@ -11,9 +11,9 @@ OBS_GRID_STAT_INPUT_DATATYPE = NETCDF OBS_IS_PROB = False SCRUB_STAGING_DIR = False #### Output -OUTPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}_{ENV[STEP]}/METplus_output +OUTPUT_BASE = {ENV[job_num_work_dir]} LOG_DIR = {OUTPUT_BASE}/logs -TMP_DIR = {OUTPUT_BASE}/tmp +TMP_DIR = {ENV[MET_TMP_DIR]} STAGING_DIR = {OUTPUT_BASE}/stage GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} GRID_STAT_OUTPUT_TEMPLATE = {ENV[RUN]}.{ENV[DATE]}/{MODEL}/{ENV[VERIF_CASE]} diff --git a/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsGFS_climoERA5_Days6_10Avg.conf b/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsGFS_climoERA5_Days6_10Avg.conf index f9674d9ff..1b5783b57 100644 --- a/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsGFS_climoERA5_Days6_10Avg.conf +++ b/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsGFS_climoERA5_Days6_10Avg.conf @@ -12,9 +12,9 @@ OBS_IS_PROB = False SCRUB_STAGING_DIR = False GRID_STAT_CLIMO_MEAN_FILE_NAME = {INPUT_BASE}/METplus_output/{ENV[RUN]}.{ENV[DATE]}/{MODEL}/{ENV[VERIF_CASE]}/days6_10_avg_{ENV[VERIF_TYPE]}_{ENV[job_name]}_init{init?fmt=%Y%m%d%H}_valid{valid?fmt=%Y%m%d%H?shift=-120H}to{valid?fmt=%Y%m%d%H}.nc #### Output -OUTPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}_{ENV[STEP]}/METplus_output +OUTPUT_BASE = {ENV[job_num_work_dir]} LOG_DIR = {OUTPUT_BASE}/logs -TMP_DIR = {OUTPUT_BASE}/tmp +TMP_DIR = {ENV[MET_TMP_DIR]} STAGING_DIR = {OUTPUT_BASE}/stage GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} GRID_STAT_OUTPUT_TEMPLATE = {ENV[RUN]}.{ENV[DATE]}/{MODEL}/{ENV[VERIF_CASE]} diff --git a/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsGFS_climoERA5_Days6_10NetCDF.conf b/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsGFS_climoERA5_Days6_10NetCDF.conf index 415a23af1..013c85c13 100644 --- a/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsGFS_climoERA5_Days6_10NetCDF.conf +++ b/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsGFS_climoERA5_Days6_10NetCDF.conf @@ -13,9 +13,9 @@ SCRUB_STAGING_DIR = False GRID_STAT_CLIMO_MEAN_FILE_NAME = {ENV[FIXevs]}/climos/{ENV[RUN]}/era5/mean_{valid?fmt=%m%d} GRID_STAT_CLIMO_STDEV_FILE_NAME = {ENV[FIXevs]}/climos/{ENV[RUN]}/era5/stdev_{valid?fmt=%m%d} #### Output -OUTPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}_{ENV[STEP]}/METplus_output +OUTPUT_BASE = {ENV[job_num_work_dir]} LOG_DIR = {OUTPUT_BASE}/logs -TMP_DIR = {OUTPUT_BASE}/tmp +TMP_DIR = {ENV[MET_TMP_DIR]} STAGING_DIR = {OUTPUT_BASE}/stage GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} GRID_STAT_OUTPUT_TEMPLATE = {ENV[RUN]}.{ENV[DATE]}/{MODEL}/{ENV[VERIF_CASE]} diff --git a/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsGFS_climoERA5_WeeklyAvg.conf b/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsGFS_climoERA5_WeeklyAvg.conf index 8bfb629e1..c2f59ab48 100644 --- a/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsGFS_climoERA5_WeeklyAvg.conf +++ b/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsGFS_climoERA5_WeeklyAvg.conf @@ -12,9 +12,9 @@ OBS_IS_PROB = False SCRUB_STAGING_DIR = False GRID_STAT_CLIMO_MEAN_FILE_NAME = {INPUT_BASE}/METplus_output/{ENV[RUN]}.{ENV[DATE]}/{MODEL}/{ENV[VERIF_CASE]}/weekly_avg_{ENV[VERIF_TYPE]}_{ENV[job_name]}_init{init?fmt=%Y%m%d%H}_valid{valid?fmt=%Y%m%d%H?shift=-168H}to{valid?fmt=%Y%m%d%H}.nc #### Output -OUTPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}_{ENV[STEP]}/METplus_output +OUTPUT_BASE = {ENV[job_num_work_dir]} LOG_DIR = {OUTPUT_BASE}/logs -TMP_DIR = {OUTPUT_BASE}/tmp +TMP_DIR = {ENV[MET_TMP_DIR]} STAGING_DIR = {OUTPUT_BASE}/stage GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} GRID_STAT_OUTPUT_TEMPLATE = {ENV[RUN]}.{ENV[DATE]}/{MODEL}/{ENV[VERIF_CASE]} diff --git a/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsGFS_climoERA5_WeeklyNetCDF.conf b/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsGFS_climoERA5_WeeklyNetCDF.conf index 242f1b0c3..69f4273e4 100644 --- a/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsGFS_climoERA5_WeeklyNetCDF.conf +++ b/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsGFS_climoERA5_WeeklyNetCDF.conf @@ -13,9 +13,9 @@ SCRUB_STAGING_DIR = False GRID_STAT_CLIMO_MEAN_FILE_NAME = {ENV[FIXevs]}/climos/{ENV[RUN]}/era5/mean_{valid?fmt=%m%d} GRID_STAT_CLIMO_STDEV_FILE_NAME = {ENV[FIXevs]}/climos/{ENV[RUN]}/era5/stdev_{valid?fmt=%m%d} #### Output -OUTPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}_{ENV[STEP]}/METplus_output +OUTPUT_BASE = {ENV[job_num_work_dir]} LOG_DIR = {OUTPUT_BASE}/logs -TMP_DIR = {OUTPUT_BASE}/tmp +TMP_DIR = {ENV[MET_TMP_DIR]} STAGING_DIR = {OUTPUT_BASE}/stage GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} GRID_STAT_OUTPUT_TEMPLATE = {ENV[RUN]}.{ENV[DATE]}/{MODEL}/{ENV[VERIF_CASE]} diff --git a/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsGFS_climoERA5_Weeks3_4Avg.conf b/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsGFS_climoERA5_Weeks3_4Avg.conf index 61d99f9d3..a460f5f3d 100644 --- a/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsGFS_climoERA5_Weeks3_4Avg.conf +++ b/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsGFS_climoERA5_Weeks3_4Avg.conf @@ -12,9 +12,9 @@ OBS_IS_PROB = False SCRUB_STAGING_DIR = False GRID_STAT_CLIMO_MEAN_FILE_NAME = {INPUT_BASE}/METplus_output/{ENV[RUN]}.{ENV[DATE]}/{MODEL}/{ENV[VERIF_CASE]}/weeks3_4_avg_{ENV[VERIF_TYPE]}_{ENV[job_name]}_init{init?fmt=%Y%m%d%H}_valid{valid?fmt=%Y%m%d%H?shift=-336H}to{valid?fmt=%Y%m%d%H}.nc #### Output -OUTPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}_{ENV[STEP]}/METplus_output +OUTPUT_BASE = {ENV[job_num_work_dir]} LOG_DIR = {OUTPUT_BASE}/logs -TMP_DIR = {OUTPUT_BASE}/tmp +TMP_DIR = {ENV[MET_TMP_DIR]} STAGING_DIR = {OUTPUT_BASE}/stage GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} GRID_STAT_OUTPUT_TEMPLATE = {ENV[RUN]}.{ENV[DATE]}/{MODEL}/{ENV[VERIF_CASE]} diff --git a/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsGFS_climoERA5_Weeks3_4NetCDF.conf b/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsGFS_climoERA5_Weeks3_4NetCDF.conf index d1e35d4ea..6f6269584 100644 --- a/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsGFS_climoERA5_Weeks3_4NetCDF.conf +++ b/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsGFS_climoERA5_Weeks3_4NetCDF.conf @@ -13,9 +13,9 @@ SCRUB_STAGING_DIR = False GRID_STAT_CLIMO_MEAN_FILE_NAME = {ENV[FIXevs]}/climos/{ENV[RUN]}/era5/mean_{valid?fmt=%m%d} GRID_STAT_CLIMO_STDEV_FILE_NAME = {ENV[FIXevs]}/climos/{ENV[RUN]}/era5/stdev_{valid?fmt=%m%d} #### Output -OUTPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}_{ENV[STEP]}/METplus_output +OUTPUT_BASE = {ENV[job_num_work_dir]} LOG_DIR = {OUTPUT_BASE}/logs -TMP_DIR = {OUTPUT_BASE}/tmp +TMP_DIR = {ENV[MET_TMP_DIR]} STAGING_DIR = {OUTPUT_BASE}/stage GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} GRID_STAT_OUTPUT_TEMPLATE = {ENV[RUN]}.{ENV[DATE]}/{MODEL}/{ENV[VERIF_CASE]} diff --git a/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsGHRSST_OSPO_DailyAvg.conf b/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsGHRSST_OSPO_DailyAvg.conf index c9cf3df41..014cffdba 100644 --- a/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsGHRSST_OSPO_DailyAvg.conf +++ b/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsGHRSST_OSPO_DailyAvg.conf @@ -11,9 +11,9 @@ OBS_GRID_STAT_INPUT_DATATYPE = NETCDF OBS_IS_PROB = False SCRUB_STAGING_DIR = False #### Output -OUTPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}_{ENV[STEP]}/METplus_output +OUTPUT_BASE = {ENV[job_num_work_dir]} LOG_DIR = {OUTPUT_BASE}/logs -TMP_DIR = {OUTPUT_BASE}/tmp +TMP_DIR = {ENV[MET_TMP_DIR]} STAGING_DIR = {OUTPUT_BASE}/stage GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} GRID_STAT_OUTPUT_TEMPLATE = {ENV[RUN]}.{ENV[DATE]}/{MODEL}/{ENV[VERIF_CASE]} diff --git a/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsGHRSST_OSPO_MonthlyAvg.conf b/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsGHRSST_OSPO_MonthlyAvg.conf index 9d0ef4601..bc11be40c 100644 --- a/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsGHRSST_OSPO_MonthlyAvg.conf +++ b/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsGHRSST_OSPO_MonthlyAvg.conf @@ -11,9 +11,9 @@ OBS_GRID_STAT_INPUT_DATATYPE = NETCDF OBS_IS_PROB = False SCRUB_STAGING_DIR = False #### Output -OUTPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}_{ENV[STEP]}/METplus_output +OUTPUT_BASE = {ENV[job_num_work_dir]} LOG_DIR = {OUTPUT_BASE}/logs -TMP_DIR = {OUTPUT_BASE}/tmp +TMP_DIR = {ENV[MET_TMP_DIR]} STAGING_DIR = {OUTPUT_BASE}/stage GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} GRID_STAT_OUTPUT_TEMPLATE = {ENV[RUN]}.{ENV[DATE]}/{MODEL}/{ENV[VERIF_CASE]} diff --git a/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsGHRSST_OSPO_WeeklyAvg.conf b/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsGHRSST_OSPO_WeeklyAvg.conf index 76512ddda..ff7abb3db 100644 --- a/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsGHRSST_OSPO_WeeklyAvg.conf +++ b/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsGHRSST_OSPO_WeeklyAvg.conf @@ -11,9 +11,9 @@ OBS_GRID_STAT_INPUT_DATATYPE = NETCDF OBS_IS_PROB = False SCRUB_STAGING_DIR = False #### Output -OUTPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}_{ENV[STEP]}/METplus_output +OUTPUT_BASE = {ENV[job_num_work_dir]} LOG_DIR = {OUTPUT_BASE}/logs -TMP_DIR = {OUTPUT_BASE}/tmp +TMP_DIR = {ENV[MET_TMP_DIR]} STAGING_DIR = {OUTPUT_BASE}/stage GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} GRID_STAT_OUTPUT_TEMPLATE = {ENV[RUN]}.{ENV[DATE]}/{MODEL}/{ENV[VERIF_CASE]} diff --git a/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsOSI-SAF_MonthlyAvg.conf b/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsOSI-SAF_MonthlyAvg.conf index a8f43731d..1d38054cd 100644 --- a/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsOSI-SAF_MonthlyAvg.conf +++ b/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsOSI-SAF_MonthlyAvg.conf @@ -11,9 +11,9 @@ OBS_GRID_STAT_INPUT_DATATYPE = NETCDF OBS_IS_PROB = False SCRUB_STAGING_DIR = False #### Output -OUTPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}_{ENV[STEP]}/METplus_output +OUTPUT_BASE = {ENV[job_num_work_dir]} LOG_DIR = {OUTPUT_BASE}/logs -TMP_DIR = {OUTPUT_BASE}/tmp +TMP_DIR = {ENV[MET_TMP_DIR]} STAGING_DIR = {OUTPUT_BASE}/stage GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} GRID_STAT_OUTPUT_TEMPLATE = {ENV[RUN]}.{ENV[DATE]}/{MODEL}/{ENV[VERIF_CASE]} diff --git a/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsOSI-SAF_WeeklyAvg.conf b/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsOSI-SAF_WeeklyAvg.conf index 7609e4872..379ac21d2 100644 --- a/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsOSI-SAF_WeeklyAvg.conf +++ b/parm/metplus_config/stats/subseasonal/grid2grid/GridStat_fcstSUBSEASONAL_obsOSI-SAF_WeeklyAvg.conf @@ -11,9 +11,9 @@ OBS_GRID_STAT_INPUT_DATATYPE = NETCDF OBS_IS_PROB = False SCRUB_STAGING_DIR = False #### Output -OUTPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}_{ENV[STEP]}/METplus_output +OUTPUT_BASE = {ENV[job_num_work_dir]} LOG_DIR = {OUTPUT_BASE}/logs -TMP_DIR = {OUTPUT_BASE}/tmp +TMP_DIR = {ENV[MET_TMP_DIR]} STAGING_DIR = {OUTPUT_BASE}/stage GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} GRID_STAT_OUTPUT_TEMPLATE = {ENV[RUN]}.{ENV[DATE]}/{MODEL}/{ENV[VERIF_CASE]} diff --git a/parm/metplus_config/stats/subseasonal/grid2grid/StatAnalysis_fcstSUBSEASONAL.conf b/parm/metplus_config/stats/subseasonal/grid2grid/StatAnalysis_fcstSUBSEASONAL.conf index cd7fe48b1..ad4a20854 100644 --- a/parm/metplus_config/stats/subseasonal/grid2grid/StatAnalysis_fcstSUBSEASONAL.conf +++ b/parm/metplus_config/stats/subseasonal/grid2grid/StatAnalysis_fcstSUBSEASONAL.conf @@ -4,9 +4,9 @@ INPUT_BASE = {ENV[DATA]} MODEL1_STAT_ANALYSIS_LOOKIN_DIR = {INPUT_BASE}/{ENV[VERIF_CASE]}_{ENV[STEP]}/METplus_output/{ENV[RUN]}.{ENV[DATE]} SCRUB_STAGING_DIR = False #### Output -OUTPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}_{ENV[STEP]}/METplus_output +OUTPUT_BASE = {ENV[job_num_work_dir]} LOG_DIR = {OUTPUT_BASE}/logs -TMP_DIR = {OUTPUT_BASE}/tmp +TMP_DIR = {ENV[MET_TMP_DIR]} STAGING_DIR = {OUTPUT_BASE}/stage STAT_ANALYSIS_OUTPUT_DIR = {OUTPUT_BASE} MODEL1_STAT_ANALYSIS_DUMP_ROW_TEMPLATE = {MODEL1}.{fcst_valid_end?fmt=%Y%m%d}/evs.stats.{MODEL1}.{ENV[RUN]}.{ENV[VERIF_CASE]}.v{fcst_valid_end?fmt=%Y%m%d}.stat diff --git a/parm/metplus_config/stats/subseasonal/grid2obs/GenEnsProd_fcstSUBSEASONAL_Days6_10NetCDF.conf b/parm/metplus_config/stats/subseasonal/grid2obs/GenEnsProd_fcstSUBSEASONAL_Days6_10NetCDF.conf index 22a901abc..fcb9b4246 100644 --- a/parm/metplus_config/stats/subseasonal/grid2obs/GenEnsProd_fcstSUBSEASONAL_Days6_10NetCDF.conf +++ b/parm/metplus_config/stats/subseasonal/grid2obs/GenEnsProd_fcstSUBSEASONAL_Days6_10NetCDF.conf @@ -9,9 +9,9 @@ SCRUB_STAGING_DIR = False MODEL = {ENV[MODEL]} OBTYPE = {ENV[MODEL]} #### Output -OUTPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}_{ENV[STEP]}/METplus_output +OUTPUT_BASE = {ENV[job_num_work_dir]} LOG_DIR = {OUTPUT_BASE}/logs -TMP_DIR = {OUTPUT_BASE}/tmp +TMP_DIR = {ENV[MET_TMP_DIR]} STAGING_DIR = {OUTPUT_BASE}/stage GEN_ENS_PROD_OUTPUT_DIR = {OUTPUT_BASE} GEN_ENS_PROD_OUTPUT_TEMPLATE = {ENV[RUN]}.{ENV[DATE]}/{MODEL}/{ENV[VERIF_CASE]}/gen_ens_prod_{ENV[VERIF_TYPE]}_{ENV[job_name]}_FHR{lead?fmt=%3H}_{valid?fmt=%Y%m%d}_{valid?fmt=%H}0000V_ens.nc diff --git a/parm/metplus_config/stats/subseasonal/grid2obs/GenEnsProd_fcstSUBSEASONAL_WeeklyNetCDF.conf b/parm/metplus_config/stats/subseasonal/grid2obs/GenEnsProd_fcstSUBSEASONAL_WeeklyNetCDF.conf index 6e9c1472e..1928f66f2 100644 --- a/parm/metplus_config/stats/subseasonal/grid2obs/GenEnsProd_fcstSUBSEASONAL_WeeklyNetCDF.conf +++ b/parm/metplus_config/stats/subseasonal/grid2obs/GenEnsProd_fcstSUBSEASONAL_WeeklyNetCDF.conf @@ -9,9 +9,9 @@ SCRUB_STAGING_DIR = False MODEL = {ENV[MODEL]} OBTYPE = {ENV[MODEL]} #### Output -OUTPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}_{ENV[STEP]}/METplus_output +OUTPUT_BASE = {ENV[job_num_work_dir]} LOG_DIR = {OUTPUT_BASE}/logs -TMP_DIR = {OUTPUT_BASE}/tmp +TMP_DIR = {ENV[MET_TMP_DIR]} STAGING_DIR = {OUTPUT_BASE}/stage GEN_ENS_PROD_OUTPUT_DIR = {OUTPUT_BASE} GEN_ENS_PROD_OUTPUT_TEMPLATE = {ENV[RUN]}.{ENV[DATE]}/{MODEL}/{ENV[VERIF_CASE]}/gen_ens_prod_{ENV[VERIF_TYPE]}_{ENV[job_name]}_FHR{lead?fmt=%3H}_{valid?fmt=%Y%m%d}_{valid?fmt=%H}0000V_ens.nc diff --git a/parm/metplus_config/stats/subseasonal/grid2obs/GenEnsProd_fcstSUBSEASONAL_Weeks3_4NetCDF.conf b/parm/metplus_config/stats/subseasonal/grid2obs/GenEnsProd_fcstSUBSEASONAL_Weeks3_4NetCDF.conf index 861c00c6e..08b14bfe0 100644 --- a/parm/metplus_config/stats/subseasonal/grid2obs/GenEnsProd_fcstSUBSEASONAL_Weeks3_4NetCDF.conf +++ b/parm/metplus_config/stats/subseasonal/grid2obs/GenEnsProd_fcstSUBSEASONAL_Weeks3_4NetCDF.conf @@ -9,9 +9,9 @@ SCRUB_STAGING_DIR = False MODEL = {ENV[MODEL]} OBTYPE = {ENV[MODEL]} #### Output -OUTPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}_{ENV[STEP]}/METplus_output +OUTPUT_BASE = {ENV[job_num_work_dir]} LOG_DIR = {OUTPUT_BASE}/logs -TMP_DIR = {OUTPUT_BASE}/tmp +TMP_DIR = {ENV[MET_TMP_DIR]} STAGING_DIR = {OUTPUT_BASE}/stage GEN_ENS_PROD_OUTPUT_DIR = {OUTPUT_BASE} GEN_ENS_PROD_OUTPUT_TEMPLATE = {ENV[RUN]}.{ENV[DATE]}/{MODEL}/{ENV[VERIF_CASE]}/gen_ens_prod_{ENV[VERIF_TYPE]}_{ENV[job_name]}_FHR{lead?fmt=%3H}_{valid?fmt=%Y%m%d}_{valid?fmt=%H}0000V_ens.nc diff --git a/parm/metplus_config/stats/subseasonal/grid2obs/PB2NC_obsPrepbufr_Weeks3_4.conf b/parm/metplus_config/stats/subseasonal/grid2obs/PB2NC_obsPrepbufr_Weeks3_4.conf index 2b6303366..d318c8b98 100644 --- a/parm/metplus_config/stats/subseasonal/grid2obs/PB2NC_obsPrepbufr_Weeks3_4.conf +++ b/parm/metplus_config/stats/subseasonal/grid2obs/PB2NC_obsPrepbufr_Weeks3_4.conf @@ -6,9 +6,9 @@ PB2NC_INPUT_TEMPLATE = prepbufr.{ENV[prepbufr]}.{valid?fmt=%Y%m%d%H} PB2NC_INPUT_DATATYPE = GRIB SCRUB_STAGING_DIR = False #### Output -OUTPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}_{ENV[STEP]}/METplus_output +OUTPUT_BASE = {ENV[job_num_work_dir]} LOG_DIR = {OUTPUT_BASE}/logs -TMP_DIR = {OUTPUT_BASE}/tmp +TMP_DIR = {ENV[MET_TMP_DIR]} STAGING_DIR = {OUTPUT_BASE}/stage PB2NC_OUTPUT_DIR = {OUTPUT_BASE} PB2NC_OUTPUT_TEMPLATE = {ENV[RUN]}.{ENV[DATE]}/prepbufr/{ENV[VERIF_CASE]}/pb2nc_{ENV[VERIF_TYPE]}_{ENV[prepbufr]}_valid{valid?fmt=%Y%m%d%H}.nc diff --git a/parm/metplus_config/stats/subseasonal/grid2obs/PointStat_fcstSUBSEASONAL_obsPrepbufr_climoERA5_Days6_10MPR.conf b/parm/metplus_config/stats/subseasonal/grid2obs/PointStat_fcstSUBSEASONAL_obsPrepbufr_climoERA5_Days6_10MPR.conf index 1c07ec94d..6fb6dbfed 100644 --- a/parm/metplus_config/stats/subseasonal/grid2obs/PointStat_fcstSUBSEASONAL_obsPrepbufr_climoERA5_Days6_10MPR.conf +++ b/parm/metplus_config/stats/subseasonal/grid2obs/PointStat_fcstSUBSEASONAL_obsPrepbufr_climoERA5_Days6_10MPR.conf @@ -13,9 +13,9 @@ SCRUB_STAGING_DIR = False POINT_STAT_CLIMO_MEAN_FILE_NAME = {ENV[FIXevs]}/climos/{ENV[RUN]}/era5/mean_{valid?fmt=%m%d} POINT_STAT_CLIMO_STDEV_FILE_NAME = {ENV[FIXevs]}/climos/{ENV[RUN]}/era5/stdev_{valid?fmt=%m%d} #### Output -OUTPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}_{ENV[STEP]}/METplus_output +OUTPUT_BASE = {ENV[job_num_work_dir]} LOG_DIR = {OUTPUT_BASE}/logs -TMP_DIR = {OUTPUT_BASE}/tmp +TMP_DIR = {ENV[MET_TMP_DIR]} STAGING_DIR = {OUTPUT_BASE}/stage POINT_STAT_OUTPUT_DIR = {OUTPUT_BASE} POINT_STAT_OUTPUT_TEMPLATE = {ENV[RUN]}.{ENV[DATE]}/{MODEL}/{ENV[VERIF_CASE]} @@ -49,7 +49,6 @@ OBTYPE = {ENV[prepbufr]} OBS_POINT_STAT_WINDOW_BEGIN = -{ENV[obs_window]} OBS_POINT_STAT_WINDOW_END = {ENV[obs_window]} POINT_STAT_OBS_QUALITY = -#POINT_STAT_OBS_QUALITY = 1,2,3,9 POINT_STAT_OFFSETS = 0 POINT_STAT_MESSAGE_TYPE_GROUP_MAP = [{key="AIRUPA";val="ADPUPA,AIRCAR,AIRCFT";}, {key="SURFACE";val="ADPSFC,SFCSHP,MSONET";}, {key="ANYAIR";val="AIRCAR,AIRCFT";}, {key="ANYSFC";val="ADPSFC,SFCSHP,ADPUPA,PROFLR,MSONET";}, {key="ONLYSF";val="ADPSFC,SFCSHP";}, {key="LANDSF";val="ADPSFC,MSONET";}, {key="WATERSF";val="SFCSHP";}] POINT_STAT_MESSAGE_TYPE = {ENV[msg_type]} diff --git a/parm/metplus_config/stats/subseasonal/grid2obs/PointStat_fcstSUBSEASONAL_obsPrepbufr_climoERA5_WeeklyMPR.conf b/parm/metplus_config/stats/subseasonal/grid2obs/PointStat_fcstSUBSEASONAL_obsPrepbufr_climoERA5_WeeklyMPR.conf index c61bc6654..781a31ab5 100644 --- a/parm/metplus_config/stats/subseasonal/grid2obs/PointStat_fcstSUBSEASONAL_obsPrepbufr_climoERA5_WeeklyMPR.conf +++ b/parm/metplus_config/stats/subseasonal/grid2obs/PointStat_fcstSUBSEASONAL_obsPrepbufr_climoERA5_WeeklyMPR.conf @@ -13,9 +13,9 @@ SCRUB_STAGING_DIR = False POINT_STAT_CLIMO_MEAN_FILE_NAME = {ENV[FIXevs]}/climos/{ENV[RUN]}/era5/mean_{valid?fmt=%m%d} POINT_STAT_CLIMO_STDEV_FILE_NAME = {ENV[FIXevs]}/climos/{ENV[RUN]}/era5/stdev_{valid?fmt=%m%d} #### Output -OUTPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}_{ENV[STEP]}/METplus_output +OUTPUT_BASE = {ENV[job_num_work_dir]} LOG_DIR = {OUTPUT_BASE}/logs -TMP_DIR = {OUTPUT_BASE}/tmp +TMP_DIR = {ENV[MET_TMP_DIR]} STAGING_DIR = {OUTPUT_BASE}/stage POINT_STAT_OUTPUT_DIR = {OUTPUT_BASE} POINT_STAT_OUTPUT_TEMPLATE = {ENV[RUN]}.{ENV[DATE]}/{MODEL}/{ENV[VERIF_CASE]} @@ -49,7 +49,6 @@ OBTYPE = {ENV[prepbufr]} OBS_POINT_STAT_WINDOW_BEGIN = -{ENV[obs_window]} OBS_POINT_STAT_WINDOW_END = {ENV[obs_window]} POINT_STAT_OBS_QUALITY = -#POINT_STAT_OBS_QUALITY = 1,2,3,9 POINT_STAT_OFFSETS = 0 POINT_STAT_MESSAGE_TYPE_GROUP_MAP = [{key="AIRUPA";val="ADPUPA,AIRCAR,AIRCFT";}, {key="SURFACE";val="ADPSFC,SFCSHP,MSONET";}, {key="ANYAIR";val="AIRCAR,AIRCFT";}, {key="ANYSFC";val="ADPSFC,SFCSHP,ADPUPA,PROFLR,MSONET";}, {key="ONLYSF";val="ADPSFC,SFCSHP";}, {key="LANDSF";val="ADPSFC,MSONET";}, {key="WATERSF";val="SFCSHP";}] POINT_STAT_MESSAGE_TYPE = {ENV[msg_type]} diff --git a/parm/metplus_config/stats/subseasonal/grid2obs/PointStat_fcstSUBSEASONAL_obsPrepbufr_climoERA5_Weeks3_4MPR.conf b/parm/metplus_config/stats/subseasonal/grid2obs/PointStat_fcstSUBSEASONAL_obsPrepbufr_climoERA5_Weeks3_4MPR.conf index ccc10f919..c29a4b8d6 100644 --- a/parm/metplus_config/stats/subseasonal/grid2obs/PointStat_fcstSUBSEASONAL_obsPrepbufr_climoERA5_Weeks3_4MPR.conf +++ b/parm/metplus_config/stats/subseasonal/grid2obs/PointStat_fcstSUBSEASONAL_obsPrepbufr_climoERA5_Weeks3_4MPR.conf @@ -13,9 +13,9 @@ SCRUB_STAGING_DIR = False POINT_STAT_CLIMO_MEAN_FILE_NAME = {ENV[FIXevs]}/climos/{ENV[RUN]}/era5/mean_{valid?fmt=%m%d} POINT_STAT_CLIMO_STDEV_FILE_NAME = {ENV[FIXevs]}/climos/{ENV[RUN]}/era5/stdev_{valid?fmt=%m%d} #### Output -OUTPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}_{ENV[STEP]}/METplus_output +OUTPUT_BASE = {ENV[job_num_work_dir]} LOG_DIR = {OUTPUT_BASE}/logs -TMP_DIR = {OUTPUT_BASE}/tmp +TMP_DIR = {ENV[MET_TMP_DIR]} STAGING_DIR = {OUTPUT_BASE}/stage POINT_STAT_OUTPUT_DIR = {OUTPUT_BASE} POINT_STAT_OUTPUT_TEMPLATE = {ENV[RUN]}.{ENV[DATE]}/{MODEL}/{ENV[VERIF_CASE]} @@ -49,7 +49,6 @@ OBTYPE = {ENV[prepbufr]} OBS_POINT_STAT_WINDOW_BEGIN = -{ENV[obs_window]} OBS_POINT_STAT_WINDOW_END = {ENV[obs_window]} POINT_STAT_OBS_QUALITY = -#POINT_STAT_OBS_QUALITY = 1,2,3,9 POINT_STAT_OFFSETS = 0 POINT_STAT_MESSAGE_TYPE_GROUP_MAP = [{key="AIRUPA";val="ADPUPA,AIRCAR,AIRCFT";}, {key="SURFACE";val="ADPSFC,SFCSHP,MSONET";}, {key="ANYAIR";val="AIRCAR,AIRCFT";}, {key="ANYSFC";val="ADPSFC,SFCSHP,ADPUPA,PROFLR,MSONET";}, {key="ONLYSF";val="ADPSFC,SFCSHP";}, {key="LANDSF";val="ADPSFC,MSONET";}, {key="WATERSF";val="SFCSHP";}] POINT_STAT_MESSAGE_TYPE = {ENV[msg_type]} diff --git a/parm/metplus_config/stats/subseasonal/grid2obs/StatAnalysis_fcstSUBSEASONAL.conf b/parm/metplus_config/stats/subseasonal/grid2obs/StatAnalysis_fcstSUBSEASONAL.conf index 1e2799b9a..313554122 100644 --- a/parm/metplus_config/stats/subseasonal/grid2obs/StatAnalysis_fcstSUBSEASONAL.conf +++ b/parm/metplus_config/stats/subseasonal/grid2obs/StatAnalysis_fcstSUBSEASONAL.conf @@ -4,9 +4,9 @@ INPUT_BASE = {ENV[DATA]} MODEL1_STAT_ANALYSIS_LOOKIN_DIR = {INPUT_BASE}/{ENV[VERIF_CASE]}_{ENV[STEP]}/METplus_output/{ENV[RUN]}.{ENV[DATE]}/{MODEL1}/{ENV[VERIF_CASE]}/stat_analysis_* SCRUB_STAGING_DIR = False #### Output -OUTPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}_{ENV[STEP]}/METplus_output +OUTPUT_BASE = {ENV[job_num_work_dir]} LOG_DIR = {OUTPUT_BASE}/logs -TMP_DIR = {OUTPUT_BASE}/tmp +TMP_DIR = {ENV[MET_TMP_DIR]} STAGING_DIR = {OUTPUT_BASE}/stage STAT_ANALYSIS_OUTPUT_DIR = {OUTPUT_BASE} MODEL1_STAT_ANALYSIS_DUMP_ROW_TEMPLATE = {MODEL1}.{fcst_valid_end?fmt=%Y%m%d}/evs.stats.{MODEL1}.{ENV[RUN]}.{ENV[VERIF_CASE]}.v{fcst_valid_end?fmt=%Y%m%d}.stat diff --git a/parm/metplus_config/stats/subseasonal/grid2obs/StatAnalysis_fcstSUBSEASONAL_obsPrepbufr_Days6_10MPRtoSL1L2.conf b/parm/metplus_config/stats/subseasonal/grid2obs/StatAnalysis_fcstSUBSEASONAL_obsPrepbufr_Days6_10MPRtoSL1L2.conf index 2e30ff4bf..ea2f6c940 100644 --- a/parm/metplus_config/stats/subseasonal/grid2obs/StatAnalysis_fcstSUBSEASONAL_obsPrepbufr_Days6_10MPRtoSL1L2.conf +++ b/parm/metplus_config/stats/subseasonal/grid2obs/StatAnalysis_fcstSUBSEASONAL_obsPrepbufr_Days6_10MPRtoSL1L2.conf @@ -1,12 +1,12 @@ [config] #### Input -INPUT_BASE = {ENV[DATA]} -MODEL1_STAT_ANALYSIS_LOOKIN_DIR = {INPUT_BASE}/{ENV[VERIF_CASE]}_{ENV[STEP]}/METplus_output/{ENV[RUN]}.{ENV[DATE]}/{MODEL1}/{ENV[VERIF_CASE]}/days6_10_avg_prepbufr_Days6_10Avg_TempAnom2m* +INPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}_{ENV[STEP]} +MODEL1_STAT_ANALYSIS_LOOKIN_DIR = {ENV[COMOUT]}/{ENV[RUN]}.{ENV[DATE]}/{MODEL1}/{ENV[VERIF_CASE]}/days6_10_avg_prepbufr_Days6_10Avg_TempAnom2m* SCRUB_STAGING_DIR = False #### Output -OUTPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}_{ENV[STEP]}/METplus_output +OUTPUT_BASE = {ENV[job_num_work_dir]} LOG_DIR = {OUTPUT_BASE}/logs -TMP_DIR = {OUTPUT_BASE}/tmp +TMP_DIR = {ENV[MET_TMP_DIR]} STAGING_DIR = {OUTPUT_BASE}/stage STAT_ANALYSIS_OUTPUT_DIR = {OUTPUT_BASE} MODEL1_STAT_ANALYSIS_OUT_STAT_TEMPLATE = {ENV[RUN]}.{ENV[DATE]}/{MODEL1}/{ENV[VERIF_CASE]}/stat_analysis_fcst{MODEL1}_obsprepbufr_{MODEL1_OBTYPE}_{ENV[job_name]}_SL1L2_valid{valid?fmt=%Y%m%d%H?shift=-120H}to{valid?fmt=%Y%m%d%H}.stat diff --git a/parm/metplus_config/stats/subseasonal/grid2obs/StatAnalysis_fcstSUBSEASONAL_obsPrepbufr_WeeklyMPRtoSL1L2.conf b/parm/metplus_config/stats/subseasonal/grid2obs/StatAnalysis_fcstSUBSEASONAL_obsPrepbufr_WeeklyMPRtoSL1L2.conf index 69f2b8fae..49adc7fef 100644 --- a/parm/metplus_config/stats/subseasonal/grid2obs/StatAnalysis_fcstSUBSEASONAL_obsPrepbufr_WeeklyMPRtoSL1L2.conf +++ b/parm/metplus_config/stats/subseasonal/grid2obs/StatAnalysis_fcstSUBSEASONAL_obsPrepbufr_WeeklyMPRtoSL1L2.conf @@ -1,12 +1,12 @@ [config] #### Input -INPUT_BASE = {ENV[DATA]} -MODEL1_STAT_ANALYSIS_LOOKIN_DIR = {INPUT_BASE}/{ENV[VERIF_CASE]}_{ENV[STEP]}/METplus_output/{ENV[RUN]}.{ENV[DATE]}/{MODEL1}/{ENV[VERIF_CASE]}/weekly_avg_prepbufr_WeeklyAvg_TempAnom2m* +INPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}_{ENV[STEP]} +MODEL1_STAT_ANALYSIS_LOOKIN_DIR = {ENV[COMOUT]}/{ENV[RUN]}.{ENV[DATE]}/{MODEL1}/{ENV[VERIF_CASE]}/weekly_avg_prepbufr_WeeklyAvg_TempAnom2m* SCRUB_STAGING_DIR = False #### Output -OUTPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}_{ENV[STEP]}/METplus_output +OUTPUT_BASE = {ENV[job_num_work_dir]} LOG_DIR = {OUTPUT_BASE}/logs -TMP_DIR = {OUTPUT_BASE}/tmp +TMP_DIR = {ENV[MET_TMP_DIR]} STAGING_DIR = {OUTPUT_BASE}/stage STAT_ANALYSIS_OUTPUT_DIR = {OUTPUT_BASE} MODEL1_STAT_ANALYSIS_OUT_STAT_TEMPLATE = {ENV[RUN]}.{ENV[DATE]}/{MODEL1}/{ENV[VERIF_CASE]}/stat_analysis_fcst{MODEL1}_obsprepbufr_{MODEL1_OBTYPE}_{ENV[job_name]}_SL1L2_valid{valid?fmt=%Y%m%d%H?shift=-168H}to{valid?fmt=%Y%m%d%H}.stat diff --git a/parm/metplus_config/stats/subseasonal/grid2obs/StatAnalysis_fcstSUBSEASONAL_obsPrepbufr_Weeks3_4MPRtoSL1L2.conf b/parm/metplus_config/stats/subseasonal/grid2obs/StatAnalysis_fcstSUBSEASONAL_obsPrepbufr_Weeks3_4MPRtoSL1L2.conf index 98fdfb765..b5c4e6d15 100644 --- a/parm/metplus_config/stats/subseasonal/grid2obs/StatAnalysis_fcstSUBSEASONAL_obsPrepbufr_Weeks3_4MPRtoSL1L2.conf +++ b/parm/metplus_config/stats/subseasonal/grid2obs/StatAnalysis_fcstSUBSEASONAL_obsPrepbufr_Weeks3_4MPRtoSL1L2.conf @@ -1,12 +1,12 @@ [config] #### Input -INPUT_BASE = {ENV[DATA]} -MODEL1_STAT_ANALYSIS_LOOKIN_DIR = {INPUT_BASE}/{ENV[VERIF_CASE]}_{ENV[STEP]}/METplus_output/{ENV[RUN]}.{ENV[DATE]}/{MODEL1}/{ENV[VERIF_CASE]}/weeks3_4_avg_prepbufr_Weeks3_4Avg_TempAnom2m* +INPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}_{ENV[STEP]} +MODEL1_STAT_ANALYSIS_LOOKIN_DIR = {ENV[COMOUT]}/{ENV[RUN]}.{ENV[DATE]}/{MODEL1}/{ENV[VERIF_CASE]}/weeks3_4_avg_prepbufr_Weeks3_4Avg_TempAnom2m* SCRUB_STAGING_DIR = False #### Output -OUTPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}_{ENV[STEP]}/METplus_output +OUTPUT_BASE = {ENV[job_num_work_dir]} LOG_DIR = {OUTPUT_BASE}/logs -TMP_DIR = {OUTPUT_BASE}/tmp +TMP_DIR = {ENV[MET_TMP_DIR]} STAGING_DIR = {OUTPUT_BASE}/stage STAT_ANALYSIS_OUTPUT_DIR = {OUTPUT_BASE} MODEL1_STAT_ANALYSIS_OUT_STAT_TEMPLATE = {ENV[RUN]}.{ENV[DATE]}/{MODEL1}/{ENV[VERIF_CASE]}/stat_analysis_fcst{MODEL1}_obsprepbufr_{MODEL1_OBTYPE}_{ENV[job_name]}_SL1L2_valid{valid?fmt=%Y%m%d%H?shift=-336H}to{valid?fmt=%Y%m%d%H}.stat diff --git a/parm/metplus_config/stats/subseasonal/grid2obs/StatAnalysis_fcstSUBSEASONAL_obsPrepbufr_climoERA5_Days6_10MPRtoSAL1L2.conf b/parm/metplus_config/stats/subseasonal/grid2obs/StatAnalysis_fcstSUBSEASONAL_obsPrepbufr_climoERA5_Days6_10MPRtoSAL1L2.conf index 9b281d11c..2a8afd124 100644 --- a/parm/metplus_config/stats/subseasonal/grid2obs/StatAnalysis_fcstSUBSEASONAL_obsPrepbufr_climoERA5_Days6_10MPRtoSAL1L2.conf +++ b/parm/metplus_config/stats/subseasonal/grid2obs/StatAnalysis_fcstSUBSEASONAL_obsPrepbufr_climoERA5_Days6_10MPRtoSAL1L2.conf @@ -1,12 +1,12 @@ [config] #### Input -INPUT_BASE = {ENV[DATA]} -MODEL1_STAT_ANALYSIS_LOOKIN_DIR = {INPUT_BASE}/{ENV[VERIF_CASE]}_{ENV[STEP]}/METplus_output/{ENV[RUN]}.{ENV[DATE]}/{MODEL1}/{ENV[VERIF_CASE]}/days6_10_avg_prepbufr_Days6_10Avg_Temp2m* +INPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}_{ENV[STEP]} +MODEL1_STAT_ANALYSIS_LOOKIN_DIR = {ENV[COMOUT]}/{ENV[RUN]}.{ENV[DATE]}/{MODEL1}/{ENV[VERIF_CASE]}/days6_10_avg_prepbufr_Days6_10Avg_Temp2m* SCRUB_STAGING_DIR = False #### Output -OUTPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}_{ENV[STEP]}/METplus_output +OUTPUT_BASE = {ENV[job_num_work_dir]} LOG_DIR = {OUTPUT_BASE}/logs -TMP_DIR = {OUTPUT_BASE}/tmp +TMP_DIR = {ENV[MET_TMP_DIR]} STAGING_DIR = {OUTPUT_BASE}/stage STAT_ANALYSIS_OUTPUT_DIR = {OUTPUT_BASE} MODEL1_STAT_ANALYSIS_OUT_STAT_TEMPLATE = {ENV[RUN]}.{ENV[DATE]}/{MODEL1}/{ENV[VERIF_CASE]}/stat_analysis_fcst{MODEL1}_obsprepbufr_{MODEL1_OBTYPE}_{ENV[job_name]}_SAL1L2_valid{valid?fmt=%Y%m%d%H?shift=-120H}to{valid?fmt=%Y%m%d%H}.stat diff --git a/parm/metplus_config/stats/subseasonal/grid2obs/StatAnalysis_fcstSUBSEASONAL_obsPrepbufr_climoERA5_WeeklyMPRtoSAL1L2.conf b/parm/metplus_config/stats/subseasonal/grid2obs/StatAnalysis_fcstSUBSEASONAL_obsPrepbufr_climoERA5_WeeklyMPRtoSAL1L2.conf index 7b30d1fa1..a5d55de0d 100644 --- a/parm/metplus_config/stats/subseasonal/grid2obs/StatAnalysis_fcstSUBSEASONAL_obsPrepbufr_climoERA5_WeeklyMPRtoSAL1L2.conf +++ b/parm/metplus_config/stats/subseasonal/grid2obs/StatAnalysis_fcstSUBSEASONAL_obsPrepbufr_climoERA5_WeeklyMPRtoSAL1L2.conf @@ -1,12 +1,12 @@ [config] #### Input -INPUT_BASE = {ENV[DATA]} -MODEL1_STAT_ANALYSIS_LOOKIN_DIR = {INPUT_BASE}/{ENV[VERIF_CASE]}_{ENV[STEP]}/METplus_output/{ENV[RUN]}.{ENV[DATE]}/{MODEL1}/{ENV[VERIF_CASE]}/weekly_avg_prepbufr_WeeklyAvg_Temp2m* +INPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}_{ENV[STEP]} +MODEL1_STAT_ANALYSIS_LOOKIN_DIR = {ENV[COMOUT]}/{ENV[RUN]}.{ENV[DATE]}/{MODEL1}/{ENV[VERIF_CASE]}/weekly_avg_prepbufr_WeeklyAvg_Temp2m* SCRUB_STAGING_DIR = False #### Output -OUTPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}_{ENV[STEP]}/METplus_output +OUTPUT_BASE = {ENV[job_num_work_dir]} LOG_DIR = {OUTPUT_BASE}/logs -TMP_DIR = {OUTPUT_BASE}/tmp +TMP_DIR = {ENV[MET_TMP_DIR]} STAGING_DIR = {OUTPUT_BASE}/stage STAT_ANALYSIS_OUTPUT_DIR = {OUTPUT_BASE} MODEL1_STAT_ANALYSIS_OUT_STAT_TEMPLATE = {ENV[RUN]}.{ENV[DATE]}/{MODEL1}/{ENV[VERIF_CASE]}/stat_analysis_fcst{MODEL1}_obsprepbufr_{MODEL1_OBTYPE}_{ENV[job_name]}_SAL1L2_valid{valid?fmt=%Y%m%d%H?shift=-168H}to{valid?fmt=%Y%m%d%H}.stat diff --git a/parm/metplus_config/stats/subseasonal/grid2obs/StatAnalysis_fcstSUBSEASONAL_obsPrepbufr_climoERA5_Weeks3_4MPRtoSAL1L2.conf b/parm/metplus_config/stats/subseasonal/grid2obs/StatAnalysis_fcstSUBSEASONAL_obsPrepbufr_climoERA5_Weeks3_4MPRtoSAL1L2.conf index e59f551f3..71b68ee36 100644 --- a/parm/metplus_config/stats/subseasonal/grid2obs/StatAnalysis_fcstSUBSEASONAL_obsPrepbufr_climoERA5_Weeks3_4MPRtoSAL1L2.conf +++ b/parm/metplus_config/stats/subseasonal/grid2obs/StatAnalysis_fcstSUBSEASONAL_obsPrepbufr_climoERA5_Weeks3_4MPRtoSAL1L2.conf @@ -1,12 +1,12 @@ [config] #### Input -INPUT_BASE = {ENV[DATA]} -MODEL1_STAT_ANALYSIS_LOOKIN_DIR = {INPUT_BASE}/{ENV[VERIF_CASE]}_{ENV[STEP]}/METplus_output/{ENV[RUN]}.{ENV[DATE]}/{MODEL1}/{ENV[VERIF_CASE]}/weeks3_4_avg_prepbufr_Weeks3_4Avg_Temp2m* +INPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}_{ENV[STEP]} +MODEL1_STAT_ANALYSIS_LOOKIN_DIR = {ENV[COMOUT]}/{ENV[RUN]}.{ENV[DATE]}/{MODEL1}/{ENV[VERIF_CASE]}/weeks3_4_avg_prepbufr_Weeks3_4Avg_Temp2m* SCRUB_STAGING_DIR = False #### Output -OUTPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}_{ENV[STEP]}/METplus_output +OUTPUT_BASE = {ENV[job_num_work_dir]} LOG_DIR = {OUTPUT_BASE}/logs -TMP_DIR = {OUTPUT_BASE}/tmp +TMP_DIR = {ENV[MET_TMP_DIR]} STAGING_DIR = {OUTPUT_BASE}/stage STAT_ANALYSIS_OUTPUT_DIR = {OUTPUT_BASE} MODEL1_STAT_ANALYSIS_OUT_STAT_TEMPLATE = {ENV[RUN]}.{ENV[DATE]}/{MODEL1}/{ENV[VERIF_CASE]}/stat_analysis_fcst{MODEL1}_obsprepbufr_{MODEL1_OBTYPE}_{ENV[job_name]}_SAL1L2_valid{valid?fmt=%Y%m%d%H?shift=-336H}to{valid?fmt=%Y%m%d%H}.stat diff --git a/scripts/stats/subseasonal/exevs_subseasonal_grid2grid_stats.sh b/scripts/stats/subseasonal/exevs_subseasonal_grid2grid_stats.sh index a9bf84079..b6e31ebb0 100755 --- a/scripts/stats/subseasonal/exevs_subseasonal_grid2grid_stats.sh +++ b/scripts/stats/subseasonal/exevs_subseasonal_grid2grid_stats.sh @@ -260,6 +260,8 @@ for group in reformat_data assemble_data generate_stats gather_stats; do nc=$((nc+1)) done fi + python $USHevs/subseasonal/subseasonal_copy_job_dir_output.py + export err=$?; err_chk done diff --git a/scripts/stats/subseasonal/exevs_subseasonal_grid2obs_stats.sh b/scripts/stats/subseasonal/exevs_subseasonal_grid2obs_stats.sh index 48af87b1a..8fa02d52a 100755 --- a/scripts/stats/subseasonal/exevs_subseasonal_grid2obs_stats.sh +++ b/scripts/stats/subseasonal/exevs_subseasonal_grid2obs_stats.sh @@ -177,6 +177,8 @@ for group in reformat_data assemble_data generate_stats gather_stats; do nc=$((nc+1)) done fi + python $USHevs/subseasonal/subseasonal_copy_job_dir_output.py + export err=$?; err_chk done diff --git a/ush/subseasonal/create_METplus_subseasonal_output_dirs.py b/ush/subseasonal/create_METplus_subseasonal_output_dirs.py index 1fbf361c6..30dfe2935 100644 --- a/ush/subseasonal/create_METplus_subseasonal_output_dirs.py +++ b/ush/subseasonal/create_METplus_subseasonal_output_dirs.py @@ -76,10 +76,9 @@ working_output_base_dir = os.path.join(DATA, VERIF_CASE_STEP, 'METplus_output') working_dir_list.append(working_output_base_dir) - working_dir_list.append(os.path.join(working_output_base_dir, 'confs')) - working_dir_list.append(os.path.join(working_output_base_dir, 'logs')) - working_dir_list.append(os.path.join(working_output_base_dir, 'tmp')) - working_dir_list.append(os.path.join(working_output_base_dir, 'stage')) + working_dir_list.append( + os.path.join(working_output_base_dir, 'job_work_dir') + ) date_dt = start_date_dt while date_dt <= end_date_dt: for model in model_list: diff --git a/ush/subseasonal/subseasonal_copy_job_dir_output.py b/ush/subseasonal/subseasonal_copy_job_dir_output.py new file mode 100644 index 000000000..299d55357 --- /dev/null +++ b/ush/subseasonal/subseasonal_copy_job_dir_output.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +''' +Program Name: subseasonal_copy_job_dir_output.py +Contact(s): Shannon Shields +Abstract: This script is run by all stats and plots scripts in scripts/. + This copies MPMD working directory output to common DATA directory. +''' + +import os +import glob +import subseasonal_util as sub_util + +print("BEGIN: "+os.path.basename(__file__)) + +#Read in environment variables +DATA = os.environ['DATA'] +RUN = os.environ['RUN'] +VERIF_CASE = os.environ['VERIF_CASE'] +STEP = os.environ['STEP'] +COMPONENT = os.environ['COMPONENT'] +JOB_GROUP = os.environ['JOB_GROUP'] +if STEP == 'plots': + NDAYS = os.environ['NDAYS'] + +# Copy files to desired location +if STEP == 'stats': + job_work_JOB_GROUP_dir = os.path.join( + DATA, f"{VERIF_CASE}_{STEP}", 'METplus_output', + 'job_work_dir', JOB_GROUP + ) + if JOB_GROUP == 'gather_stats': + job_wildcard_dir = os.path.join( + job_work_JOB_GROUP_dir, 'job*', '*.*', '*' + ) + else: + job_wildcard_dir = os.path.join( + job_work_JOB_GROUP_dir, 'job*', f"{RUN}.*", '*', VERIF_CASE, '*' + ) +elif STEP == 'plots': + job_work_JOB_GROUP_dir = os.path.join( + DATA, f"{VERIF_CASE}_{STEP}", 'plot_output', + 'job_work_dir', JOB_GROUP + ) + if JOB_GROUP == 'condense_stats': + job_wildcard_dir = os.path.join( + job_work_JOB_GROUP_dir, 'job*', f"{RUN}.*", '*', + f"last{NDAYS}days", '*', '*', '*', 'condensed_stats*' + ) + elif JOB_GROUP == 'filter_stats': + job_wildcard_dir = os.path.join( + job_work_JOB_GROUP_dir, 'job*', f"{RUN}.*", '*', + f"last{NDAYS}days", '*', '*', '*', 'fcst*.stat' + ) + elif JOB_GROUP == 'make_plots': + job_wildcard_dir = os.path.join( + job_work_JOB_GROUP_dir, 'job*', f"{RUN}.*", '*', + f"last{NDAYS}days", '*', '*', '*', '*', '*.png' + ) +output_file_JOB_list = glob.glob(job_wildcard_dir) +if STEP == 'plots' and JOB_GROUP == 'make_plots': + job_wildcard_dir2 = os.path.join( + job_work_JOB_GROUP_dir, 'job*', f"{RUN}.*", '*', + f"last{NDAYS}days", '*', '*', '*', '*', '*.gif' + ) + output_file_JOB_list = ( + output_file_JOB_list + + glob.glob(job_wildcard_dir2) + ) +for output_file_JOB in sorted(output_file_JOB_list, key=len): + output_file_end_path = output_file_JOB.partition( + job_work_JOB_GROUP_dir+'/' + )[2].partition('/')[2] + if STEP == 'stats': + output_file_DATA = os.path.join( + DATA, f"{VERIF_CASE}_{STEP}", 'METplus_output', + output_file_end_path + ) + elif STEP == 'plots': + output_file_DATA = os.path.join( + DATA, f"{VERIF_CASE}_{STEP}", 'plot_output', + output_file_end_path + ) + if not os.path.exists(output_file_DATA): + sub_util.copy_file(output_file_JOB, output_file_DATA) + +print("END: "+os.path.basename(__file__)) diff --git a/ush/subseasonal/subseasonal_stats_grid2grid_create_daily_avg.py b/ush/subseasonal/subseasonal_stats_grid2grid_create_daily_avg.py index 138c1f7b8..6b5f44057 100644 --- a/ush/subseasonal/subseasonal_stats_grid2grid_create_daily_avg.py +++ b/ush/subseasonal/subseasonal_stats_grid2grid_create_daily_avg.py @@ -35,6 +35,7 @@ valid_hr_inc = os.environ['valid_hr_inc'] fhr_list = os.environ['fhr_list'].split(',') fhr_inc = '12' +job_num_work_dir = os.environ['job_num_work_dir'] # Process run time agruments if len(sys.argv) != 4: @@ -54,9 +55,9 @@ DATAROOT_file_format = sys.argv[2] COMIN_file_format = sys.argv[3] -# Set input and output directories -output_dir = os.path.join(DATA, VERIF_CASE+'_'+STEP, 'METplus_output', - RUN+'.'+DATE) +# Set DATA output directory +full_path_DATA = os.path.join(DATA, VERIF_CASE+'_'+STEP, 'METplus_output', + RUN+'.'+DATE, MODEL, VERIF_CASE) # Create daily average files print("\nCreating daily average files") @@ -71,14 +72,17 @@ daily_avg_day_start = 1 daily_avg_day = daily_avg_day_start while daily_avg_day <= daily_avg_day_end: + full_path_job_num_work_dir = os.path.join( + job_num_work_dir, RUN+'.'+DATE, + MODEL, VERIF_CASE + ) daily_avg_file_list = [] daily_avg_day_fhr_end = int(daily_avg_day * 24) daily_avg_day_fhr_start = daily_avg_day_fhr_end - 24 daily_avg_day_init = (daily_avg_valid_end - datetime.timedelta(days=daily_avg_day)) daily_avg_day_fhr = daily_avg_day_fhr_start - output_file = os.path.join(output_dir, MODEL, - VERIF_CASE, + output_file = os.path.join(full_path_job_num_work_dir, 'daily_avg_' +VERIF_TYPE+'_'+job_name+'_init' +daily_avg_day_init.strftime('%Y%m%d%H') @@ -87,6 +91,9 @@ .strftime('%Y%m%d%H')+'to' +daily_avg_valid_end\ .strftime('%Y%m%d%H')+'.nc') + output_file_DATA = os.path.join( + full_path_DATA, output_file.rpartition('/')[2] + ) if os.path.exists(output_file): os.remove(output_file) daily_avg_fcst_sum = 0 @@ -163,6 +170,8 @@ if len(daily_avg_fcst_file_list) == expected_nfiles \ and len(daily_avg_obs_file_list) == expected_nfiles: print("Output File: "+output_file) + if not os.path.exists(full_path_job_num_work_dir): + os.makedirs(full_path_job_num_work_dir) output_file_data = netcdf.Dataset(output_file, 'w', format='NETCDF3_CLASSIC') for attr in input_file_data.ncattrs(): diff --git a/ush/subseasonal/subseasonal_stats_grid2grid_create_daily_reformat_job_scripts.py b/ush/subseasonal/subseasonal_stats_grid2grid_create_daily_reformat_job_scripts.py index 336db11e6..b0f99db33 100644 --- a/ush/subseasonal/subseasonal_stats_grid2grid_create_daily_reformat_job_scripts.py +++ b/ush/subseasonal/subseasonal_stats_grid2grid_create_daily_reformat_job_scripts.py @@ -142,6 +142,7 @@ for model_idx in range(len(model_list)): job_env_dict['MODEL'] = model_list[model_idx] njobs = (int(njobs) + 1) + job_env_dict['job_num'] = str(njobs) # Create job file job_file = os.path.join(JOB_GROUP_jobs_dir, 'job'+str(njobs)) print("Creating job script: "+job_file) @@ -149,6 +150,15 @@ job.write('#!/bin/bash\n') job.write('set -x\n') job.write('\n') + # Create job working directory + job_env_dict['job_num_work_dir'] = os.path.join( + DATA, f"{VERIF_CASE}_{STEP}", 'METplus_output', + 'job_work_dir', JOB_GROUP, + f"job{job_env_dict['job_num']}" + ) + job_env_dict['MET_TMP_DIR'] = os.path.join( + job_env_dict['job_num_work_dir'], 'tmp' + ) # Set any environment variables for special cases # Do file checks all_truth_file_exist = False @@ -157,7 +167,7 @@ check_model_files = True if check_model_files: (model_files_exist, valid_date_fhr_list, - model_copy_output_DATA2COMOUT_list) = ( + model_copy_output_list) = ( sub_util.check_daily_model_files(job_env_dict) ) job_env_dict['fhr_list'] = ( @@ -189,6 +199,8 @@ job.write('\n') # Write job commands if write_job_cmds: + if not os.path.exists(job_env_dict['job_num_work_dir']): + os.makedirs(job_env_dict['job_num_work_dir']) for cmd in verif_type_job_commands_list: job.write(cmd+'\n') job.write('export err=$?; err_chk'+'\n') diff --git a/ush/subseasonal/subseasonal_stats_grid2grid_create_days6_10_anomaly.py b/ush/subseasonal/subseasonal_stats_grid2grid_create_days6_10_anomaly.py index 5595d90b3..108aa8bfe 100644 --- a/ush/subseasonal/subseasonal_stats_grid2grid_create_days6_10_anomaly.py +++ b/ush/subseasonal/subseasonal_stats_grid2grid_create_days6_10_anomaly.py @@ -30,28 +30,22 @@ MODEL = os.environ['MODEL'] D6_10START = os.environ['D6_10START'] DATE = os.environ['DATE'] +var1_name = os.environ['var1_name'] +var1_levels = os.environ['var1_levels'] valid_hr_start = os.environ['valid_hr_start'] valid_hr_end = os.environ['valid_hr_end'] valid_hr_inc = os.environ['valid_hr_inc'] fhr_list = os.environ['CORRECT_LEAD_SEQ'].split(',') +job_num_work_dir = os.environ['job_num_work_dir'] -# Process run time arguments -if len(sys.argv) != 3: - print("FATAL ERROR: Not given correct number of run time arguments..." - +os.path.basename(__file__)+" VARNAME_VARLEVEL FILE_FORMAT") +# Check variable settings +if ' ' in var1_levels or ',' in var1_levels: + print("ERROR: Cannot accept list of levels") sys.exit(1) -else: - if '_' not in sys.argv[1]: - print("FATAL ERROR: variable and level runtime argument formatted " - +"incorrectly, be sure to separate variable and level with " - +"an underscore (_), example HGT_P500") - sys.exit(1) - else: - var_level = sys.argv[1] - print("Using var_level = "+var_level) - file_format = sys.argv[2] -output_var_level = (var_level.split('_')[0]+'_ANOM_' - +var_level.split('_')[-1]) + +# Set variable to make anomalies +var_level = f"{var1_name}_{var1_levels}" +output_var_level = f"{var1_name}_ANOM_{var1_levels}" # Create fcst and obs anomaly data STARTDATE_dt = datetime.datetime.strptime( @@ -65,11 +59,44 @@ valid_date_dt = STARTDATE_dt fhr = fhr_start while valid_date_dt <= ENDDATE_dt and fhr <= fhr_end: + # Set full paths for dates + full_path_job_num_work_dir = os.path.join( + job_num_work_dir, RUN+'.' + +ENDDATE_dt.strftime('%Y%m%d'), + MODEL, VERIF_CASE + ) + full_path_DATA = os.path.join( + DATA, VERIF_CASE+'_'+STEP, 'METplus_output', + RUN+'.'+ENDDATE_dt.strftime('%Y%m%d'), + MODEL, VERIF_CASE + ) init_date_dt = valid_date_dt - datetime.timedelta(hours=fhr) - input_file = sub_util.format_filler( - file_format, valid_date_dt, init_date_dt, str(fhr), {} + input_file_name = ( + f"grid_stat_{VERIF_TYPE}_{job_name}_{str(fhr).zfill(2)}0000L_" + +f"{valid_date_dt:%Y%m%d_%H}0000V_pairs.nc" + ) + # Check possible input files + check_input_file_list = [ + os.path.join(full_path_job_num_work_dir, input_file_name), + os.path.join(full_path_DATA, input_file_name) + ] + found_input = False + for check_input_file in check_input_file_list: + if os.path.exists(check_input_file): + input_file = check_input_file + found_input = True + break + # Set output file + output_file = os.path.join( + full_path_job_num_work_dir, 'anomaly_' + +VERIF_TYPE+'_'+job_name+'_init' + +init_date_dt.strftime('%Y%m%d%H')+'_' + +'fhr'+str(fhr).zfill(3)+'.nc' + ) + output_file_DATA = os.path.join( + full_path_DATA, output_file.rpartition('/')[2] ) - if os.path.exists(input_file): + if found_input: print("\nInput file: "+input_file) input_file_data = netcdf.Dataset(input_file) input_file_data_var_list = list(input_file_data.variables.keys()) @@ -81,16 +108,9 @@ print("WARNING: "+input_file+" does not contain any " +"climo variable cannot make anomaly data") else: - output_dir = os.path.join(DATA, VERIF_CASE+'_'+STEP, - 'METplus_output', - RUN+'.' - +ENDDATE_dt.strftime('%Y%m%d'), - MODEL, VERIF_CASE) - output_file = os.path.join(output_dir, 'anomaly_' - +VERIF_TYPE+'_'+job_name+'_init' - +init_date_dt.strftime('%Y%m%d%H')+'_' - +'fhr'+str(fhr).zfill(3)+'.nc') print("Output File: "+output_file) + if not os.path.exists(full_path_job_num_work_dir): + os.makedirs(full_path_job_num_work_dir) if os.path.exists(output_file): os.remove(output_file) output_file_data = netcdf.Dataset(output_file, 'w', diff --git a/ush/subseasonal/subseasonal_stats_grid2grid_create_days6_10_avg.py b/ush/subseasonal/subseasonal_stats_grid2grid_create_days6_10_avg.py index 379693145..748d77076 100644 --- a/ush/subseasonal/subseasonal_stats_grid2grid_create_days6_10_avg.py +++ b/ush/subseasonal/subseasonal_stats_grid2grid_create_days6_10_avg.py @@ -35,6 +35,7 @@ valid_hr_inc = os.environ['valid_hr_inc'] fhr_list = os.environ['fhr_list'].split(',') fhr_inc = '12' +job_num_work_dir = os.environ['job_num_work_dir'] # Process run time arguments if len(sys.argv) != 4: @@ -54,9 +55,9 @@ DATAROOT_file_format = sys.argv[2] COMIN_file_format = sys.argv[3] -# Set input and output directories -output_dir = os.path.join(DATA, VERIF_CASE+'_'+STEP, 'METplus_output', - RUN+'.'+DATE) +# Set DATA output directory +full_path_DATA = os.path.join(DATA, VERIF_CASE+'_'+STEP, 'METplus_output', + RUN+'.'+DATE, MODEL, VERIF_CASE) # Create Days 6-10 average files print("\nCreating Days 6-10 average files") @@ -71,14 +72,17 @@ days_avg_day_start = 10 days_avg_day = days_avg_day_start while days_avg_day <= days_avg_day_end: + full_path_job_num_work_dir = os.path.join( + job_num_work_dir, RUN+'.'+DATE, + MODEL, VERIF_CASE + ) days_avg_day_fhr_end = int(days_avg_day * 24) days_avg_file_list = [] days_avg_day_fhr_start = days_avg_day_fhr_end - 120 days_avg_day_init = (days_avg_valid_end - datetime.timedelta(days=days_avg_day)) days_avg_day_fhr = days_avg_day_fhr_start - output_file = os.path.join(output_dir, MODEL, - VERIF_CASE, + output_file = os.path.join(full_path_job_num_work_dir, 'days6_10_avg_' +VERIF_TYPE+'_'+job_name+'_init' +days_avg_day_init.strftime('%Y%m%d%H') @@ -87,6 +91,9 @@ .strftime('%Y%m%d%H')+'to' +days_avg_valid_end\ .strftime('%Y%m%d%H')+'.nc') + output_file_DATA = os.path.join( + full_path_DATA, output_file.rpartition('/')[2] + ) if os.path.exists(output_file): os.remove(output_file) days_avg_fcst_sum = 0 @@ -182,6 +189,8 @@ if len(days_avg_fcst_file_list) >= expected_nfiles \ and len(days_avg_obs_file_list) >= expected_nfiles: print("Output File: "+output_file) + if not os.path.exists(full_path_job_num_work_dir): + os.makedirs(full_path_job_num_work_dir) output_file_data = netcdf.Dataset(output_file, 'w', format='NETCDF3_CLASSIC') for attr in input_file_data.ncattrs(): diff --git a/ush/subseasonal/subseasonal_stats_grid2grid_create_days6_10_reformat_job_scripts.py b/ush/subseasonal/subseasonal_stats_grid2grid_create_days6_10_reformat_job_scripts.py index c328d7e5b..e8daba807 100644 --- a/ush/subseasonal/subseasonal_stats_grid2grid_create_days6_10_reformat_job_scripts.py +++ b/ush/subseasonal/subseasonal_stats_grid2grid_create_days6_10_reformat_job_scripts.py @@ -83,19 +83,7 @@ sub_util.python_command( 'subseasonal_stats_grid2grid' '_create_days6_10_anomaly.py', - ['HGT_P500', - os.path.join( - '$DATA', - '${VERIF_CASE}_${STEP}', - 'METplus_output', - '${RUN}.$DATE', - '$MODEL', '$VERIF_CASE', - 'grid_stat_${VERIF_TYPE}_' - +'${job_name}_' - +'{lead?fmt=%2H}0000L_' - +'{valid?fmt=%Y%m%d}_' - +'{valid?fmt=%H}0000V_pairs.nc' - )] + [] )]}, }, 'temp': {}, @@ -121,19 +109,7 @@ sub_util.python_command( 'subseasonal_stats_grid2grid' '_create_days6_10_anomaly.py', - ['HGT_P500', - os.path.join( - '$DATA', - '${VERIF_CASE}_${STEP}', - 'METplus_output', - '${RUN}.$DATE', - '$MODEL', '$VERIF_CASE', - 'grid_stat_${VERIF_TYPE}_' - +'${job_name}_' - +'{lead?fmt=%2H}0000L_' - +'{valid?fmt=%Y%m%d}_' - +'{valid?fmt=%H}0000V_pairs.nc' - )] + [] )]}, }, 'temp': {}, @@ -156,19 +132,7 @@ sub_util.python_command( 'subseasonal_stats_grid2grid' '_create_days6_10_anomaly.py', - ['TMP_Z2', - os.path.join( - '$DATA', - '${VERIF_CASE}_${STEP}', - 'METplus_output', - '${RUN}.$DATE', - '$MODEL', '$VERIF_CASE', - 'grid_stat_${VERIF_TYPE}_' - +'${job_name}_' - +'{lead?fmt=%2H}0000L_' - +'{valid?fmt=%Y%m%d}_' - +'{valid?fmt=%H}0000V_pairs.nc' - )] + [] )]}, }, 'pres_lvls': {}, @@ -245,6 +209,7 @@ for model_idx in range(len(model_list)): job_env_dict['MODEL'] = model_list[model_idx] njobs = (int(njobs) + 1) + job_env_dict['job_num'] = str(njobs) # Create job file job_file = os.path.join(JOB_GROUP_jobs_dir, 'job'+str(njobs)) print("Creating job script: "+job_file) @@ -252,6 +217,15 @@ job.write('#!/bin/bash\n') job.write('set -x\n') job.write('\n') + # Create job working directory + job_env_dict['job_num_work_dir'] = os.path.join( + DATA, f"{VERIF_CASE}_{STEP}", 'METplus_output', + 'job_work_dir', JOB_GROUP, + f"job{job_env_dict['job_num']}" + ) + job_env_dict['MET_TMP_DIR'] = os.path.join( + job_env_dict['job_num_work_dir'], 'tmp' + ) # Set any environment variables for special cases # Do file checks all_truth_file_exist = False @@ -260,7 +234,7 @@ check_model_files = True if check_model_files: (model_files_exist, valid_date_fhr_list, - model_copy_output_DATA2COMOUT_list) = ( + model_copy_output_list) = ( sub_util.check_days6_10_model_files(job_env_dict) ) job_env_dict['fhr_list'] = ( @@ -280,7 +254,7 @@ check_truth_files = False if check_truth_files: (all_truth_file_exist, - truth_copy_output_DATA2COMOUT_list) = ( + truth_copy_output_list) = ( sub_util.check_days6_10_truth_files(job_env_dict) ) if model_files_exist and all_truth_file_exist: @@ -299,6 +273,8 @@ job.write('\n') # Write job commands if write_job_cmds: + if not os.path.exists(job_env_dict['job_num_work_dir']): + os.makedirs(job_env_dict['job_num_work_dir']) for cmd in verif_type_job_commands_list: job.write(cmd+'\n') job.write('export err=$?; err_chk'+'\n') diff --git a/ush/subseasonal/subseasonal_stats_grid2grid_create_job_scripts.py b/ush/subseasonal/subseasonal_stats_grid2grid_create_job_scripts.py index 95a0cacef..de8096a38 100644 --- a/ush/subseasonal/subseasonal_stats_grid2grid_create_job_scripts.py +++ b/ush/subseasonal/subseasonal_stats_grid2grid_create_job_scripts.py @@ -898,6 +898,7 @@ for model_idx in range(len(model_list)): job_env_dict['MODEL'] = model_list[model_idx] njobs+=1 + job_env_dict['job_num'] = str(njobs) # Create job file job_file = os.path.join(JOB_GROUP_jobs_dir, 'job'+str(njobs)) print("Creating job script: "+job_file) @@ -905,6 +906,15 @@ job.write('#!/bin/bash\n') job.write('set -x\n') job.write('\n') + # Create job working directory + job_env_dict['job_num_work_dir'] = os.path.join( + DATA, f"{VERIF_CASE}_{STEP}", 'METplus_output', + 'job_work_dir', JOB_GROUP, + f"job{job_env_dict['job_num']}" + ) + job_env_dict['MET_TMP_DIR'] = os.path.join( + job_env_dict['job_num_work_dir'], 'tmp' + ) # Set any environment variables for special cases # Do file checks all_truth_file_exist = False @@ -913,7 +923,7 @@ check_model_files = True if check_model_files: (model_files_exist, valid_date_fhr_list, - model_copy_output_DATA2COMOUT_list) = ( + model_copy_output_list) = ( sub_util.check_model_files(job_env_dict) ) job_env_dict['fhr_list'] = ( @@ -948,6 +958,8 @@ job.write('\n') # Write job commands if write_job_cmds: + if not os.path.exists(job_env_dict['job_num_work_dir']): + os.makedirs(job_env_dict['job_num_work_dir']) for cmd in verif_type_job_commands_list: job.write(cmd+'\n') job.write('export err=$?; err_chk'+'\n') @@ -1043,6 +1055,7 @@ for model_idx in range(len(model_list)): job_env_dict['MODEL'] = model_list[model_idx] njobs+=1 + job_env_dict['job_num'] = str(njobs) # Create job file job_file = os.path.join(JOB_GROUP_jobs_dir, 'job'+str(njobs)) print("Creating job script: "+job_file) @@ -1050,6 +1063,15 @@ job.write('#!/bin/bash\n') job.write('set -x\n') job.write('\n') + # Create job working directory + job_env_dict['job_num_work_dir'] = os.path.join( + DATA, f"{VERIF_CASE}_{STEP}", 'METplus_output', + 'job_work_dir', JOB_GROUP, + f"job{job_env_dict['job_num']}" + ) + job_env_dict['MET_TMP_DIR'] = os.path.join( + job_env_dict['job_num_work_dir'], 'tmp' + ) # Set any environment variables for special cases # Write environment variables for name, value in job_env_dict.items(): @@ -1063,6 +1085,8 @@ write_job_cmds = False # Write job commands if write_job_cmds: + if not os.path.exists(job_env_dict['job_num_work_dir']): + os.makedirs(job_env_dict['job_num_work_dir']) for cmd in gather_stats_jobs_dict['commands']: job.write(cmd+'\n') job.write('export err=$?; err_chk'+'\n') diff --git a/ush/subseasonal/subseasonal_stats_grid2grid_create_monthly_avg.py b/ush/subseasonal/subseasonal_stats_grid2grid_create_monthly_avg.py index b68ecd413..8b138165a 100644 --- a/ush/subseasonal/subseasonal_stats_grid2grid_create_monthly_avg.py +++ b/ush/subseasonal/subseasonal_stats_grid2grid_create_monthly_avg.py @@ -35,6 +35,7 @@ valid_hr_inc = os.environ['valid_hr_inc'] fhr_list = os.environ['fhr_list'].split(',') fhr_inc = '12' +job_num_work_dir = os.environ['job_num_work_dir'] # Process run time arguments if len(sys.argv) != 4: @@ -54,9 +55,9 @@ DATAROOT_file_format = sys.argv[2] COMIN_file_format = sys.argv[3] -# Set input and output directories -output_dir = os.path.join(DATA, VERIF_CASE+'_'+STEP, 'METplus_output', - RUN+'.'+DATE) +# Set DATA output directory +full_path_DATA = os.path.join(DATA, VERIF_CASE+'_'+STEP, 'METplus_output', + RUN+'.'+DATE, MODEL, VERIF_CASE) # Create monthly average files print("\nCreating monthly average files") @@ -71,14 +72,17 @@ monthly_avg_day_start = 30 monthly_avg_day = monthly_avg_day_start while monthly_avg_day <= monthly_avg_day_end: + full_path_job_num_work_dir = os.path.join( + job_num_work_dir, RUN+'.'+DATE, + MODEL, VERIF_CASE + ) monthly_avg_day_fhr_end = int(monthly_avg_day * 24) monthly_avg_file_list = [] monthly_avg_day_fhr_start = monthly_avg_day_fhr_end - 720 monthly_avg_day_init = (monthly_avg_valid_end - datetime.timedelta(days=monthly_avg_day)) monthly_avg_day_fhr = monthly_avg_day_fhr_start - output_file = os.path.join(output_dir, MODEL, - VERIF_CASE, + output_file = os.path.join(full_path_job_num_work_dir, 'monthly_avg_' +VERIF_TYPE+'_'+job_name+'_init' +monthly_avg_day_init.strftime('%Y%m%d%H') @@ -87,6 +91,9 @@ .strftime('%Y%m%d%H')+'to' +monthly_avg_valid_end\ .strftime('%Y%m%d%H')+'.nc') + output_file_DATA = os.path.join( + full_path_DATA, output_file.rpartition('/')[2] + ) if os.path.exists(output_file): os.remove(output_file) monthly_avg_fcst_sum = 0 @@ -163,6 +170,8 @@ if len(monthly_avg_fcst_file_list) >= expected_nfiles \ and len(monthly_avg_obs_file_list) >= expected_nfiles: print("Output File: "+output_file) + if not os.path.exists(full_path_job_num_work_dir): + os.makedirs(full_path_job_num_work_dir) output_file_data = netcdf.Dataset(output_file, 'w', format='NETCDF3_CLASSIC') for attr in input_file_data.ncattrs(): diff --git a/ush/subseasonal/subseasonal_stats_grid2grid_create_monthly_reformat_job_scripts.py b/ush/subseasonal/subseasonal_stats_grid2grid_create_monthly_reformat_job_scripts.py index 948de7e61..c7c305555 100644 --- a/ush/subseasonal/subseasonal_stats_grid2grid_create_monthly_reformat_job_scripts.py +++ b/ush/subseasonal/subseasonal_stats_grid2grid_create_monthly_reformat_job_scripts.py @@ -153,6 +153,7 @@ for model_idx in range(len(model_list)): job_env_dict['MODEL'] = model_list[model_idx] njobs = (int(njobs) + 1) + job_env_dict['job_num'] = str(njobs) # Create job file job_file = os.path.join(JOB_GROUP_jobs_dir, 'job'+str(njobs)) print("Creating job script: "+job_file) @@ -160,6 +161,15 @@ job.write('#!/bin/bash\n') job.write('set -x\n') job.write('\n') + # Create job working directory + job_env_dict['job_num_work_dir'] = os.path.join( + DATA, f"{VERIF_CASE}_{STEP}", 'METplus_output', + 'job_work_dir', JOB_GROUP, + f"job{job_env_dict['job_num']}" + ) + job_env_dict['MET_TMP_DIR'] = os.path.join( + job_env_dict['job_num_work_dir'], 'tmp' + ) # Set any environment variables for special cases # Do file checks all_truth_file_exist = False @@ -168,7 +178,7 @@ check_model_files = True if check_model_files: (model_files_exist, valid_date_fhr_list, - model_copy_output_DATA2COMOUT_list) = ( + model_copy_output_list) = ( sub_util.check_monthly_model_files(job_env_dict) ) job_env_dict['fhr_list'] = ( @@ -200,6 +210,8 @@ job.write('\n') # Write job commands if write_job_cmds: + if not os.path.exists(job_env_dict['job_num_work_dir']): + os.makedirs(job_env_dict['job_num_work_dir']) for cmd in verif_type_job_commands_list: job.write(cmd+'\n') job.write('export err=$?; err_chk'+'\n') diff --git a/ush/subseasonal/subseasonal_stats_grid2grid_create_week4_reformat_job_scripts.py b/ush/subseasonal/subseasonal_stats_grid2grid_create_week4_reformat_job_scripts.py index 063958823..9f76d7b96 100644 --- a/ush/subseasonal/subseasonal_stats_grid2grid_create_week4_reformat_job_scripts.py +++ b/ush/subseasonal/subseasonal_stats_grid2grid_create_week4_reformat_job_scripts.py @@ -154,8 +154,8 @@ job_env_dict['valid_hr_end'] = date_dt.strftime('%H') for model_idx in range(len(model_list)): job_env_dict['MODEL'] = model_list[model_idx] - #njobs+=1 njobs = (int(njobs) + 1) + job_env_dict['job_num'] = str(njobs) # Create job file job_file = os.path.join(JOB_GROUP_jobs_dir, 'job'+str(njobs)) print("Creating job script: "+job_file) @@ -163,6 +163,15 @@ job.write('#!/bin/bash\n') job.write('set -x\n') job.write('\n') + # Create job working directory + job_env_dict['job_num_work_dir'] = os.path.join( + DATA, f"{VERIF_CASE}_{STEP}", 'METplus_output', + 'job_work_dir', JOB_GROUP, + f"job{job_env_dict['job_num']}" + ) + job_env_dict['MET_TMP_DIR'] = os.path.join( + job_env_dict['job_num_work_dir'], 'tmp' + ) # Set any environment variables for special cases # Do file checks all_truth_file_exist = False @@ -171,7 +180,7 @@ check_model_files = True if check_model_files: (model_files_exist, valid_date_fhr_list, - model_copy_output_DATA2COMOUT_list) = ( + model_copy_output_list) = ( sub_util.check_weekly_model_files(job_env_dict) ) job_env_dict['fhr_list'] = ( @@ -191,7 +200,7 @@ check_truth_files = False if check_truth_files: (all_truth_file_exist, - truth_copy_output_DATA2COMOUT_list) = ( + truth_copy_output_list) = ( sub_util.check_weekly_truth_files(job_env_dict) ) if model_files_exist and all_truth_file_exist: @@ -210,6 +219,8 @@ job.write('\n') # Write job commands if write_job_cmds: + if not os.path.exists(job_env_dict['job_num_work_dir']): + os.makedirs(job_env_dict['job_num_work_dir']) for cmd in verif_type_job_commands_list: job.write(cmd+'\n') job.write('export err=$?; err_chk'+'\n') diff --git a/ush/subseasonal/subseasonal_stats_grid2grid_create_weekly_anomaly.py b/ush/subseasonal/subseasonal_stats_grid2grid_create_weekly_anomaly.py index c7cbaac76..57ee75598 100644 --- a/ush/subseasonal/subseasonal_stats_grid2grid_create_weekly_anomaly.py +++ b/ush/subseasonal/subseasonal_stats_grid2grid_create_weekly_anomaly.py @@ -30,28 +30,22 @@ MODEL = os.environ['MODEL'] WEEKLYSTART = os.environ['WEEKLYSTART'] DATE = os.environ['DATE'] +var1_name = os.environ['var1_name'] +var1_levels = os.environ['var1_levels'] valid_hr_start = os.environ['valid_hr_start'] valid_hr_end = os.environ['valid_hr_end'] valid_hr_inc = os.environ['valid_hr_inc'] fhr_list = os.environ['CORRECT_LEAD_SEQ'].split(',') +job_num_work_dir = os.environ['job_num_work_dir'] -# Process run time arguments -if len(sys.argv) != 3: - print("FATAL ERROR: Not given correct number of run time arguments..." - +os.path.basename(__file__)+" VARNAME_VARLEVEL FILE_FORMAT") +# Check variable settings +if ' ' in var1_levels or ',' in var1_levels: + print("ERROR: Cannot accept list of levels") sys.exit(1) -else: - if '_' not in sys.argv[1]: - print("FATAL ERROR: variable and level runtime argument formatted " - +"incorrectly, be sure to separate variable and level with " - +"an underscore (_), example HGT_P500") - sys.exit(1) - else: - var_level = sys.argv[1] - print("Using var_level = "+var_level) - file_format = sys.argv[2] -output_var_level = (var_level.split('_')[0]+'_ANOM_' - +var_level.split('_')[-1]) + +# Set variable to make anomalies +var_level = f"{var1_name}_{var1_levels}" +output_var_level = f"{var1_name}_ANOM_{var1_levels}" # Create fcst and obs anomaly data STARTDATE_dt = datetime.datetime.strptime( @@ -65,11 +59,44 @@ valid_date_dt = STARTDATE_dt fhr = fhr_start while valid_date_dt <= ENDDATE_dt and fhr <= fhr_end: + # Set full paths for dates + full_path_job_num_work_dir = os.path.join( + job_num_work_dir, RUN+'.' + +ENDDATE_dt.strftime('%Y%m%d'), + MODEL, VERIF_CASE + ) + full_path_DATA = os.path.join( + DATA, VERIF_CASE+'_'+STEP, 'METplus_output', + RUN+'.'+ENDDATE_dt.strftime('%Y%m%d'), + MODEL, VERIF_CASE + ) init_date_dt = valid_date_dt - datetime.timedelta(hours=fhr) - input_file = sub_util.format_filler( - file_format, valid_date_dt, init_date_dt, str(fhr), {} + input_file_name = ( + f"grid_stat_{VERIF_TYPE}_{job_name}_{str(fhr).zfill(2)}0000L_" + +f"{valid_date_dt:%Y%m%d_%H}0000V_pairs.nc" + ) + # Check possible input files + check_input_file_list = [ + os.path.join(full_path_job_num_work_dir, input_file_name), + os.path.join(full_path_DATA, input_file_name) + ] + found_input = False + for check_input_file in check_input_file_list: + if os.path.exists(check_input_file): + input_file = check_input_file + found_input = True + break + # Set output file + output_file = os.path.join( + full_path_job_num_work_dir, 'anomaly_' + +VERIF_TYPE+'_'+job_name+'_init' + +init_date_dt.strftime('%Y%m%d%H')+'_' + +'fhr'+str(fhr).zfill(3)+'.nc' + ) + output_file_DATA = os.path.join( + full_path_DATA, output_file.rpartition('/')[2] ) - if os.path.exists(input_file): + if found_input: print("\nInput file: "+input_file) input_file_data = netcdf.Dataset(input_file) input_file_data_var_list = list(input_file_data.variables.keys()) @@ -81,16 +108,9 @@ print("WARNING: "+input_file+" does not contain any " +"climo variable cannot make anomaly data") else: - output_dir = os.path.join(DATA, VERIF_CASE+'_'+STEP, - 'METplus_output', - RUN+'.' - +ENDDATE_dt.strftime('%Y%m%d'), - MODEL, VERIF_CASE) - output_file = os.path.join(output_dir, 'anomaly_' - +VERIF_TYPE+'_'+job_name+'_init' - +init_date_dt.strftime('%Y%m%d%H')+'_' - +'fhr'+str(fhr).zfill(3)+'.nc') print("Output File: "+output_file) + if not os.path.exists(full_path_job_num_work_dir): + os.makedirs(full_path_job_num_work_dir) if os.path.exists(output_file): os.remove(output_file) output_file_data = netcdf.Dataset(output_file, 'w', diff --git a/ush/subseasonal/subseasonal_stats_grid2grid_create_weekly_avg.py b/ush/subseasonal/subseasonal_stats_grid2grid_create_weekly_avg.py index 2523ed955..349eb88a7 100644 --- a/ush/subseasonal/subseasonal_stats_grid2grid_create_weekly_avg.py +++ b/ush/subseasonal/subseasonal_stats_grid2grid_create_weekly_avg.py @@ -35,6 +35,7 @@ valid_hr_inc = os.environ['valid_hr_inc'] fhr_list = os.environ['fhr_list'].split(',') fhr_inc = '12' +job_num_work_dir = os.environ['job_num_work_dir'] # Process run time arguments if len(sys.argv) != 4: @@ -54,9 +55,9 @@ DATAROOT_file_format = sys.argv[2] COMIN_file_format = sys.argv[3] -# Set input and output directories -output_dir = os.path.join(DATA, VERIF_CASE+'_'+STEP, 'METplus_output', - RUN+'.'+DATE) +# Set DATA output directory +full_path_DATA = os.path.join(DATA, VERIF_CASE+'_'+STEP, 'METplus_output', + RUN+'.'+DATE, MODEL, VERIF_CASE) # Create weekly average files print("\nCreating weekly average files") @@ -71,14 +72,17 @@ weekly_avg_day_start = 7 weekly_avg_day = weekly_avg_day_start while weekly_avg_day <= weekly_avg_day_end: + full_path_job_num_work_dir = os.path.join( + job_num_work_dir, RUN+'.'+DATE, + MODEL, VERIF_CASE + ) weekly_avg_day_fhr_end = int(weekly_avg_day * 24) weekly_avg_file_list = [] weekly_avg_day_fhr_start = weekly_avg_day_fhr_end - 168 weekly_avg_day_init = (weekly_avg_valid_end - datetime.timedelta(days=weekly_avg_day)) weekly_avg_day_fhr = weekly_avg_day_fhr_start - output_file = os.path.join(output_dir, MODEL, - VERIF_CASE, + output_file = os.path.join(full_path_job_num_work_dir, 'weekly_avg_' +VERIF_TYPE+'_'+job_name+'_init' +weekly_avg_day_init.strftime('%Y%m%d%H') @@ -87,6 +91,9 @@ .strftime('%Y%m%d%H')+'to' +weekly_avg_valid_end\ .strftime('%Y%m%d%H')+'.nc') + output_file_DATA = os.path.join( + full_path_DATA, output_file.rpartition('/')[2] + ) if os.path.exists(output_file): os.remove(output_file) weekly_avg_fcst_sum = 0 @@ -182,6 +189,8 @@ if len(weekly_avg_fcst_file_list) >= expected_nfiles \ and len(weekly_avg_obs_file_list) >= expected_nfiles: print("Output File: "+output_file) + if not os.path.exists(full_path_job_num_work_dir): + os.makedirs(full_path_job_num_work_dir) output_file_data = netcdf.Dataset(output_file, 'w', format='NETCDF3_CLASSIC') for attr in input_file_data.ncattrs(): diff --git a/ush/subseasonal/subseasonal_stats_grid2grid_create_weekly_reformat_job_scripts.py b/ush/subseasonal/subseasonal_stats_grid2grid_create_weekly_reformat_job_scripts.py index 57a1b9e76..7b368d3ff 100644 --- a/ush/subseasonal/subseasonal_stats_grid2grid_create_weekly_reformat_job_scripts.py +++ b/ush/subseasonal/subseasonal_stats_grid2grid_create_weekly_reformat_job_scripts.py @@ -83,19 +83,7 @@ sub_util.python_command( 'subseasonal_stats_grid2grid' '_create_weekly_anomaly.py', - ['HGT_P500', - os.path.join( - '$DATA', - '${VERIF_CASE}_${STEP}', - 'METplus_output', - '${RUN}.$DATE', - '$MODEL', '$VERIF_CASE', - 'grid_stat_${VERIF_TYPE}_' - +'${job_name}_' - +'{lead?fmt=%2H}0000L_' - +'{valid?fmt=%Y%m%d}_' - +'{valid?fmt=%H}0000V_pairs.nc' - )] + [] )]}, }, 'temp': {}, @@ -121,19 +109,7 @@ sub_util.python_command( 'subseasonal_stats_grid2grid' '_create_weekly_anomaly.py', - ['HGT_P500', - os.path.join( - '$DATA', - '${VERIF_CASE}_${STEP}', - 'METplus_output', - '${RUN}.$DATE', - '$MODEL', '$VERIF_CASE', - 'grid_stat_${VERIF_TYPE}_' - +'${job_name}_' - +'{lead?fmt=%2H}0000L_' - +'{valid?fmt=%Y%m%d}_' - +'{valid?fmt=%H}0000V_pairs.nc' - )] + [] )]}, }, 'temp': {}, @@ -156,19 +132,7 @@ sub_util.python_command( 'subseasonal_stats_grid2grid' '_create_weekly_anomaly.py', - ['TMP_Z2', - os.path.join( - '$DATA', - '${VERIF_CASE}_${STEP}', - 'METplus_output', - '${RUN}.$DATE', - '$MODEL', '$VERIF_CASE', - 'grid_stat_${VERIF_TYPE}_' - +'${job_name}_' - +'{lead?fmt=%2H}0000L_' - +'{valid?fmt=%Y%m%d}_' - +'{valid?fmt=%H}0000V_pairs.nc' - )] + [] )]}, }, 'pres_lvls': {}, @@ -267,6 +231,7 @@ for model_idx in range(len(model_list)): job_env_dict['MODEL'] = model_list[model_idx] njobs = (int(njobs) + 1) + job_env_dict['job_num'] = str(njobs) # Create job file job_file = os.path.join(JOB_GROUP_jobs_dir, 'job'+str(njobs)) print("Creating job script: "+job_file) @@ -274,6 +239,15 @@ job.write('#!/bin/bash\n') job.write('set -x\n') job.write('\n') + # Create job working directory + job_env_dict['job_num_work_dir'] = os.path.join( + DATA, f"{VERIF_CASE}_{STEP}", 'METplus_output', + 'job_work_dir', JOB_GROUP, + f"job{job_env_dict['job_num']}" + ) + job_env_dict['MET_TMP_DIR'] = os.path.join( + job_env_dict['job_num_work_dir'], 'tmp' + ) # Set any environment variables for special cases # Do file checks all_truth_file_exist = False @@ -282,7 +256,7 @@ check_model_files = True if check_model_files: (model_files_exist, valid_date_fhr_list, - model_copy_output_DATA2COMOUT_list) = ( + model_copy_output_list) = ( sub_util.check_weekly_model_files(job_env_dict) ) job_env_dict['fhr_list'] = ( @@ -302,7 +276,7 @@ check_truth_files = False if check_truth_files: (all_truth_file_exist, - truth_copy_output_DATA2COMOUT_list) = ( + truth_copy_output_list) = ( sub_util.check_weekly_truth_files(job_env_dict) ) if model_files_exist and all_truth_file_exist: @@ -321,6 +295,8 @@ job.write('\n') # Write job commands if write_job_cmds: + if not os.path.exists(job_env_dict['job_num_work_dir']): + os.makedirs(job_env_dict['job_num_work_dir']) for cmd in verif_type_job_commands_list: job.write(cmd+'\n') job.write('export err=$?; err_chk'+'\n') diff --git a/ush/subseasonal/subseasonal_stats_grid2grid_create_weeks3_4_anomaly.py b/ush/subseasonal/subseasonal_stats_grid2grid_create_weeks3_4_anomaly.py index 8c69d4a84..fcae2f1bb 100644 --- a/ush/subseasonal/subseasonal_stats_grid2grid_create_weeks3_4_anomaly.py +++ b/ush/subseasonal/subseasonal_stats_grid2grid_create_weeks3_4_anomaly.py @@ -30,28 +30,22 @@ MODEL = os.environ['MODEL'] W3_4START = os.environ['W3_4START'] DATE = os.environ['DATE'] +var1_name = os.environ['var1_name'] +var1_levels = os.environ['var1_levels'] valid_hr_start = os.environ['valid_hr_start'] valid_hr_end = os.environ['valid_hr_end'] valid_hr_inc = os.environ['valid_hr_inc'] fhr_list = os.environ['CORRECT_LEAD_SEQ'].split(',') +job_num_work_dir = os.environ['job_num_work_dir'] -# Process run time arguments -if len(sys.argv) != 3: - print("FATAL ERROR: Not given correct number of run time arguments..." - +os.path.basename(__file__)+" VARNAME_VARLEVEL FILE_FORMAT") +# Check variable settings +if ' ' in var1_levels or ',' in var1_levels: + print("ERROR: Cannot accept list of levels") sys.exit(1) -else: - if '_' not in sys.argv[1]: - print("FATAL ERROR: variable and level runtime argument formatted " - +"incorrectly, be sure to separate variable and level with " - +"an underscore (_), example HGT_P500") - sys.exit(1) - else: - var_level = sys.argv[1] - print("Using var_level = "+var_level) - file_format = sys.argv[2] -output_var_level = (var_level.split('_')[0]+'_ANOM_' - +var_level.split('_')[-1]) + +# Set variable to make anomalies +var_level = f"{var1_name}_{var1_levels}" +output_var_level = f"{var1_name}_ANOM_{var1_levels}" # Create fcst and obs anomaly data STARTDATE_dt = datetime.datetime.strptime( @@ -65,11 +59,44 @@ valid_date_dt = STARTDATE_dt fhr = fhr_start while valid_date_dt <= ENDDATE_dt and fhr <= fhr_end: + # Set full paths for dates + full_path_job_num_work_dir = os.path.join( + job_num_work_dir, RUN+'.' + +ENDDATE_dt.strftime('%Y%m%d'), + MODEL, VERIF_CASE + ) + full_path_DATA = os.path.join( + DATA, VERIF_CASE+'_'+STEP, 'METplus_output', + RUN+'.'+ENDDATE_dt.strftime('%Y%m%d'), + MODEL, VERIF_CASE + ) init_date_dt = valid_date_dt - datetime.timedelta(hours=fhr) - input_file = sub_util.format_filler( - file_format, valid_date_dt, init_date_dt, str(fhr), {} + input_file_name = ( + f"grid_stat_{VERIF_TYPE}_{job_name}_{str(fhr).zfill(2)}0000L_" + +f"{valid_date_dt:%Y%m%d_%H}0000V_pairs.nc" + ) + # Check possible input files + check_input_file_list = [ + os.path.join(full_path_job_num_work_dir, input_file_name), + os.path.join(full_path_DATA, input_file_name) + ] + found_input = False + for check_input_file in check_input_file_list: + if os.path.exists(check_input_file): + input_file = check_input_file + found_input = True + break + # Set output file + output_file = os.path.join( + full_path_job_num_work_dir, 'anomaly_' + +VERIF_TYPE+'_'+job_name+'_init' + +init_date_dt.strftime('%Y%m%d%H')+'_' + +'fhr'+str(fhr).zfill(3)+'.nc' + ) + output_file_DATA = os.path.join( + full_path_DATA, output_file.rpartition('/')[2] ) - if os.path.exists(input_file): + if found_input: print("\nInput file: "+input_file) input_file_data = netcdf.Dataset(input_file) input_file_data_var_list = list(input_file_data.variables.keys()) @@ -81,16 +108,9 @@ print("WARNING: "+input_file+" does not contain any " +"climo variable cannot make anomaly data") else: - output_dir = os.path.join(DATA, VERIF_CASE+'_'+STEP, - 'METplus_output', - RUN+'.' - +ENDDATE_dt.strftime('%Y%m%d'), - MODEL, VERIF_CASE) - output_file = os.path.join(output_dir, 'anomaly_' - +VERIF_TYPE+'_'+job_name+'_init' - +init_date_dt.strftime('%Y%m%d%H')+'_' - +'fhr'+str(fhr).zfill(3)+'.nc') print("Output File: "+output_file) + if not os.path.exists(full_path_job_num_work_dir): + os.makedirs(full_path_job_num_work_dir) if os.path.exists(output_file): os.remove(output_file) output_file_data = netcdf.Dataset(output_file, 'w', diff --git a/ush/subseasonal/subseasonal_stats_grid2grid_create_weeks3_4_avg.py b/ush/subseasonal/subseasonal_stats_grid2grid_create_weeks3_4_avg.py index 0ec7ba452..555bb8b0f 100644 --- a/ush/subseasonal/subseasonal_stats_grid2grid_create_weeks3_4_avg.py +++ b/ush/subseasonal/subseasonal_stats_grid2grid_create_weeks3_4_avg.py @@ -35,6 +35,7 @@ valid_hr_inc = os.environ['valid_hr_inc'] fhr_list = os.environ['fhr_list'].split(',') fhr_inc = '12' +job_num_work_dir = os.environ['job_num_work_dir'] # Process run time arguments if len(sys.argv) != 4: @@ -54,9 +55,9 @@ DATAROOT_file_format = sys.argv[2] COMIN_file_format = sys.argv[3] -# Set input and output directories -output_dir = os.path.join(DATA, VERIF_CASE+'_'+STEP, 'METplus_output', - RUN+'.'+DATE) +# Set DATA output directory +full_path_DATA = os.path.join(DATA, VERIF_CASE+'_'+STEP, 'METplus_output', + RUN+'.'+DATE, MODEL, VERIF_CASE) # Create Weeks 3-4 average files print("\nCreating Weeks 3-4 average files") @@ -71,14 +72,17 @@ weeks_avg_day_start = 28 weeks_avg_day = weeks_avg_day_start while weeks_avg_day <= weeks_avg_day_end: + full_path_job_num_work_dir = os.path.join( + job_num_work_dir, RUN+'.'+DATE, + MODEL, VERIF_CASE + ) weeks_avg_day_fhr_end = int(weeks_avg_day * 24) weeks_avg_file_list = [] weeks_avg_day_fhr_start = weeks_avg_day_fhr_end - 336 weeks_avg_day_init = (weeks_avg_valid_end - datetime.timedelta(days=weeks_avg_day)) weeks_avg_day_fhr = weeks_avg_day_fhr_start - output_file = os.path.join(output_dir, MODEL, - VERIF_CASE, + output_file = os.path.join(full_path_job_num_work_dir, 'weeks3_4_avg_' +VERIF_TYPE+'_'+job_name+'_init' +weeks_avg_day_init.strftime('%Y%m%d%H') @@ -87,6 +91,9 @@ .strftime('%Y%m%d%H')+'to' +weeks_avg_valid_end\ .strftime('%Y%m%d%H')+'.nc') + output_file_DATA = os.path.join( + full_path_DATA, output_file.rpartition('/')[2] + ) if os.path.exists(output_file): os.remove(output_file) weeks_avg_fcst_sum = 0 @@ -182,6 +189,8 @@ if len(weeks_avg_fcst_file_list) >= expected_nfiles \ and len(weeks_avg_obs_file_list) >= expected_nfiles: print("Output File: "+output_file) + if not os.path.exists(full_path_job_num_work_dir): + os.makedirs(full_path_job_num_work_dir) output_file_data = netcdf.Dataset(output_file, 'w', format='NETCDF3_CLASSIC') for attr in input_file_data.ncattrs(): diff --git a/ush/subseasonal/subseasonal_stats_grid2grid_create_weeks3_4_reformat_job_scripts.py b/ush/subseasonal/subseasonal_stats_grid2grid_create_weeks3_4_reformat_job_scripts.py index 15d71633c..c89f7f54f 100644 --- a/ush/subseasonal/subseasonal_stats_grid2grid_create_weeks3_4_reformat_job_scripts.py +++ b/ush/subseasonal/subseasonal_stats_grid2grid_create_weeks3_4_reformat_job_scripts.py @@ -83,19 +83,7 @@ sub_util.python_command( 'subseasonal_stats_grid2grid' '_create_weeks3_4_anomaly.py', - ['HGT_P500', - os.path.join( - '$DATA', - '${VERIF_CASE}_${STEP}', - 'METplus_output', - '${RUN}.$DATE', - '$MODEL', '$VERIF_CASE', - 'grid_stat_${VERIF_TYPE}_' - +'${job_name}_' - +'{lead?fmt=%2H}0000L_' - +'{valid?fmt=%Y%m%d}_' - +'{valid?fmt=%H}0000V_pairs.nc' - )] + [] )]}, }, 'temp': {}, @@ -121,19 +109,7 @@ sub_util.python_command( 'subseasonal_stats_grid2grid' '_create_weeks3_4_anomaly.py', - ['HGT_P500', - os.path.join( - '$DATA', - '${VERIF_CASE}_${STEP}', - 'METplus_output', - '${RUN}.$DATE', - '$MODEL', '$VERIF_CASE', - 'grid_stat_${VERIF_TYPE}_' - +'${job_name}_' - +'{lead?fmt=%2H}0000L_' - +'{valid?fmt=%Y%m%d}_' - +'{valid?fmt=%H}0000V_pairs.nc' - )] + [] )]}, }, 'temp': {}, @@ -156,19 +132,7 @@ sub_util.python_command( 'subseasonal_stats_grid2grid' '_create_weeks3_4_anomaly.py', - ['TMP_Z2', - os.path.join( - '$DATA', - '${VERIF_CASE}_${STEP}', - 'METplus_output', - '${RUN}.$DATE', - '$MODEL', '$VERIF_CASE', - 'grid_stat_${VERIF_TYPE}_' - +'${job_name}_' - +'{lead?fmt=%2H}0000L_' - +'{valid?fmt=%Y%m%d}_' - +'{valid?fmt=%H}0000V_pairs.nc' - )] + [] )]}, }, 'pres_lvls': {}, @@ -245,6 +209,7 @@ for model_idx in range(len(model_list)): job_env_dict['MODEL'] = model_list[model_idx] njobs = (int(njobs) + 1) + job_env_dict['job_num'] = str(njobs) # Create job file job_file = os.path.join(JOB_GROUP_jobs_dir, 'job'+str(njobs)) print("Creating job script: "+job_file) @@ -252,6 +217,15 @@ job.write('#!/bin/bash\n') job.write('set -x\n') job.write('\n') + # Create job working directory + job_env_dict['job_num_work_dir'] = os.path.join( + DATA, f"{VERIF_CASE}_{STEP}", 'METplus_output', + 'job_work_dir', JOB_GROUP, + f"job{job_env_dict['job_num']}" + ) + job_env_dict['MET_TMP_DIR'] = os.path.join( + job_env_dict['job_num_work_dir'], 'tmp' + ) # Set any environment variables for special cases # Do file checks all_truth_file_exist = False @@ -260,7 +234,7 @@ check_model_files = True if check_model_files: (model_files_exist, valid_date_fhr_list, - model_copy_output_DATA2COMOUT_list) = ( + model_copy_output_list) = ( sub_util.check_weeks3_4_model_files(job_env_dict) ) job_env_dict['fhr_list'] = ( @@ -280,7 +254,7 @@ check_truth_files = False if check_truth_files: (all_truth_file_exist, - truth_copy_output_DATA2COMOUT_list) = ( + truth_copy_output_list) = ( sub_util.check_weeks3_4_truth_files(job_env_dict) ) if model_files_exist and all_truth_file_exist: @@ -299,6 +273,8 @@ job.write('\n') # Write job commands if write_job_cmds: + if not os.path.exists(job_env_dict['job_num_work_dir']): + os.makedirs(job_env_dict['job_num_work_dir']) for cmd in verif_type_job_commands_list: job.write(cmd+'\n') job.write('export err=$?; err_chk'+'\n') diff --git a/ush/subseasonal/subseasonal_stats_grid2obs_create_days6_10_anomaly.py b/ush/subseasonal/subseasonal_stats_grid2obs_create_days6_10_anomaly.py index 7d3d81304..57b5bd0c0 100644 --- a/ush/subseasonal/subseasonal_stats_grid2obs_create_days6_10_anomaly.py +++ b/ush/subseasonal/subseasonal_stats_grid2obs_create_days6_10_anomaly.py @@ -32,28 +32,18 @@ MODEL = os.environ['MODEL'] D6_10START = os.environ['D6_10START'] DATE = os.environ['DATE'] +var1_obs_name = os.environ['var1_obs_name'] +var1_obs_levels = os.environ['var1_obs_levels'] valid_hr_start = os.environ['valid_hr_start'] valid_hr_end = os.environ['valid_hr_end'] valid_hr_inc = os.environ['valid_hr_inc'] fhr_list = os.environ['CORRECT_LEAD_SEQ'].split(',') +job_num_work_dir = os.environ['job_num_work_dir'] -# Process run time arguments -if len(sys.argv) != 3: - print("FATAL ERROR: Not given correct number of run time arguments..." - +os.path.basename(__file__)+" VARNAME_VARLEVEL FILE_FORMAT") +# Check variable settings +if ' ' in var1_obs_levels or ',' in var1_obs_levels: + print("ERROR: Cannot accept list of observation levels") sys.exit(1) -else: - if '_' not in sys.argv[1]: - print("FATAL ERROR: variable and level runtime argument formatted " - +"incorrectly, be sure to separate variable and level with " - +"an underscore (_), example TMP_Z2") - sys.exit(1) - else: - var_level = sys.argv[1] - print("Using var_level = "+var_level) - file_format = sys.argv[2] -var = var_level.split('_')[0] -level = var_level.split('_')[1] # Set MET MPR columns MET_MPR_column_list = [ @@ -78,46 +68,61 @@ valid_date_dt = STARTDATE_dt fhr = fhr_start while valid_date_dt <= ENDDATE_dt and fhr <= fhr_end: + # Set full paths for dates + full_path_job_num_work_dir = os.path.join( + job_num_work_dir, RUN+'.' + +ENDDATE_dt.strftime('%Y%m%d'), + MODEL, VERIF_CASE + ) + full_path_DATA = os.path.join( + DATA, VERIF_CASE+'_'+STEP, 'METplus_output', + RUN+'.'+ENDDATE_dt.strftime('%Y%m%d'), + MODEL, VERIF_CASE + ) + full_path_COMOUT = os.path.join( + COMOUT, RUN+'.'+ENDDATE_dt.strftime('%Y%m%d'), + MODEL, VERIF_CASE + ) init_date_dt = valid_date_dt - datetime.timedelta(hours=fhr) - input_file = sub_util.format_filler( - file_format, valid_date_dt, init_date_dt, str(fhr), {} + input_file_name = ( + f"point_stat_{VERIF_TYPE}_{job_name}_{str(fhr).zfill(2)}0000L_" + +f"{valid_date_dt:%Y%m%d_%H}0000V.stat" ) - if os.path.exists(input_file): - output_dir = os.path.join(DATA, VERIF_CASE+'_'+STEP, - 'METplus_output', - RUN+'.' - +ENDDATE_dt.strftime('%Y%m%d'), - MODEL, VERIF_CASE) - output_DATA_file = os.path.join(output_dir, 'anomaly_' - +VERIF_TYPE+'_'+job_name+'_init' - +init_date_dt.strftime('%Y%m%d%H')+'_' - +'fhr'+str(fhr).zfill(3)+'.stat') - output_COMOUT_file = os.path.join(COMOUT, RUN+'.' - +ENDDATE_dt.strftime('%Y%m%d'), - MODEL, VERIF_CASE, 'anomaly_' - +VERIF_TYPE+'_'+job_name+'_init' - +init_date_dt.strftime('%Y%m%d%H')+'_' - +'fhr'+str(fhr).zfill(3)+'.stat') - if os.path.exists(output_COMOUT_file): - make_anomaly_output_file = False - sub_util.copy_file(output_COMOUT_file, output_DATA_file) - else: - if not os.path.exists(output_DATA_file): - make_anomaly_output_file = True - else: - make_anomaly_output_file = False - print(f"DATA Output File exists: {output_DATA_file}") - if SENDCOM == 'YES' \ - and sub_util.check_file_size_exists( - output_DATA_file - ): - sub_util.copy_file(output_DATA_file, - output_COMOUT_file) - else: - print(f"\nWARNING: {input_file} does not exist") + # Check possible input files + check_input_file_list = [ + os.path.join(full_path_job_num_work_dir, input_file_name), + os.path.join(full_path_DATA, input_file_name), + os.path.join(full_path_COMOUT, input_file_name) + ] + found_input = False + for check_input_file in check_input_file_list: + if os.path.exists(check_input_file): + input_file = check_input_file + found_input = True + break + # Set output file + output_file = os.path.join( + full_path_job_num_work_dir, f"anomaly_{VERIF_TYPE}_{job_name}_" + +f"init{init_date_dt:%Y%m%d%H}_fhr{str(fhr).zfill(3)}.stat" + ) + output_DATA_file = os.path.join( + full_path_DATA, output_file.rpartition('/')[2] + ) + output_COMOUT_file = os.path.join( + full_path_COMOUT, output_file.rpartition('/')[2] + ) + # Check input and output files + if os.path.exists(output_COMOUT_file): + print(f"COMOUT Output File exists: {output_COMOUT_file}") make_anomaly_output_file = False - if make_anomaly_output_file: + sub_util.copy_file(output_COMOUT_file, output_DATA_file) + else: + make_anomaly_output_file = True + if found_input and make_anomaly_output_file: print("\nInput file: "+input_file) + print(f"Output File: {output_file}") + if not os.path.exists(full_path_job_num_work_dir): + os.makedirs(full_path_job_num_work_dir) with open(input_file, 'r') as infile: input_file_header = infile.readline() sub_util.run_shell_command(['sed', '-i', '"s/ a//g"', @@ -127,7 +132,10 @@ names=MET_MPR_column_list, na_filter=False, dtype=str) input_file_var_level_df = input_file_df[ - (input_file_df['FCST_VAR'] == var) & (input_file_df['FCST_LEV'] == level) + (input_file_df['FCST_VAR'] == var1_obs_name) \ + & (input_file_df['FCST_LEV'] == var1_obs_levels) \ + & (input_file_df['OBS_VAR'] == var1_obs_name) \ + & (input_file_df['OBS_LEV'] == var1_obs_levels) ] fcst_var_level = np.array( input_file_var_level_df['FCST'].values, dtype=float @@ -145,15 +153,13 @@ output_file_df['CLIMO_MEAN'] = 'NA' output_file_df['FCST'] = fcst_anom_var_level output_file_df['OBS'] = obs_anom_var_level - output_file_df['FCST_VAR'] = var+'_ANOM' - output_file_df['OBS_VAR'] = var+'_ANOM' - print(f"DATA Output File: {output_DATA_file}") - print(f"COMOUT Output File: {output_COMOUT_file}") - output_file_df.to_csv(output_DATA_file, header=input_file_header, + output_file_df['FCST_VAR'] = f"{var1_obs_name}_ANOM" + output_file_df['OBS_VAR'] = f"{var1_obs_name}_ANOM" + output_file_df.to_csv(output_file, header=input_file_header, index=None, sep=' ', mode='w') if SENDCOM == 'YES' \ - and sub_util.check_file_exists_size(output_DATA_file): - sub_util.copy_file(output_DATA_file, output_COMOUT_file) + and sub_util.check_file_exists_size(output_file): + sub_util.copy_file(output_file, output_COMOUT_file) valid_date_dt = valid_date_dt + datetime.timedelta(hours=int(valid_hr_inc)) fhr+=int(valid_hr_inc) diff --git a/ush/subseasonal/subseasonal_stats_grid2obs_create_days6_10_assemble_job_scripts.py b/ush/subseasonal/subseasonal_stats_grid2obs_create_days6_10_assemble_job_scripts.py index 4b64a08a8..cb27d3334 100644 --- a/ush/subseasonal/subseasonal_stats_grid2obs_create_days6_10_assemble_job_scripts.py +++ b/ush/subseasonal/subseasonal_stats_grid2obs_create_days6_10_assemble_job_scripts.py @@ -83,22 +83,7 @@ 'subseasonal_stats_' 'grid2obs_create_' 'days6_10_anomaly.py', - ['TMP_Z2', - os.path.join( - '$DATA', - '${VERIF_CASE}_${STEP}', - 'METplus_output', - '${RUN}.' - +'$DATE', - '$MODEL', '$VERIF_CASE', - 'point_stat_' - +'${VERIF_TYPE}_' - +'${job_name}_' - +'{lead?fmt=%2H}0000L_' - +'{valid?fmt=%Y%m%d}_' - +'{valid?fmt=%H}0000V' - +'.stat' - )] + [] )]}, } } @@ -245,6 +230,7 @@ for model_idx in range(len(model_list)): job_env_dict['MODEL'] = model_list[model_idx] njobs = (int(njobs) + 1) + job_env_dict['job_num'] = str(njobs) # Create job file job_file = os.path.join(JOB_GROUP_jobs_dir, 'job'+str(njobs)) print("Creating job script: "+job_file) @@ -252,6 +238,15 @@ job.write('#!/bin/bash\n') job.write('set -x\n') job.write('\n') + # Create job working directory + job_env_dict['job_num_work_dir'] = os.path.join( + DATA, f"{VERIF_CASE}_{STEP}", 'METplus_output', + 'job_work_dir', JOB_GROUP, + f"job{job_env_dict['job_num']}" + ) + job_env_dict['MET_TMP_DIR'] = os.path.join( + job_env_dict['job_num_work_dir'], 'tmp' + ) # Set any environment variables for special cases if JOB_GROUP == 'assemble_data': if verif_type == 'prepbufr': @@ -280,7 +275,7 @@ check_truth_files = True if check_model_files: (model_files_exist, valid_date_fhr_list, - model_copy_output_DATA2COMOUT_list) = ( + model_copy_output_list) = ( sub_util.check_days6_10_model_files(job_env_dict) ) job_env_dict['fhr_list'] = ( @@ -291,7 +286,7 @@ job_env_dict.pop('fhr_inc') if check_truth_files: (all_truth_file_exist, - truth_copy_output_DATA2COMOUT_list) = ( + truth_copy_output_list) = ( sub_util.check_days6_10_truth_files(job_env_dict) ) if model_files_exist and all_truth_file_exist: @@ -309,14 +304,16 @@ job.write('\n') # Write job commands if write_job_cmds: + if not os.path.exists(job_env_dict['job_num_work_dir']): + os.makedirs(job_env_dict['job_num_work_dir']) for cmd in verif_type_job_commands_list: job.write(cmd+'\n') job.write('export err=$?; err_chk'+'\n') - # Copy DATA files to COMOUT restart dir + # Copy files to COMOUT restart dir # to be used in possible restart if job_env_dict['SENDCOM'] == 'YES': for model_output_file_tuple \ - in model_copy_output_DATA2COMOUT_list: + in model_copy_output_list: job.write(f'if [ -f "{model_output_file_tuple[0]}" ]; then ' +f"cp -v {model_output_file_tuple[0]} " +f"{model_output_file_tuple[1]}" diff --git a/ush/subseasonal/subseasonal_stats_grid2obs_create_days6_10_avg.py b/ush/subseasonal/subseasonal_stats_grid2obs_create_days6_10_avg.py index 40d618657..bf219df27 100644 --- a/ush/subseasonal/subseasonal_stats_grid2obs_create_days6_10_avg.py +++ b/ush/subseasonal/subseasonal_stats_grid2obs_create_days6_10_avg.py @@ -36,6 +36,7 @@ valid_hr_end = os.environ['valid_hr_end'] valid_hr_inc = os.environ['valid_hr_inc'] fhr_end = os.environ['fhr_list'].split(',')[-1] +job_num_work_dir = os.environ['job_num_work_dir'] # Process run time arguments if len(sys.argv) != 4: @@ -66,10 +67,6 @@ 'CLIMO_MEAN', 'CLIMO_STDEV', 'CLIMO_CDF' ] -# Set input and output directories -output_dir = os.path.join(DATA, VERIF_CASE+'_'+STEP, 'METplus_output', - RUN+'.'+DATE, MODEL, VERIF_CASE) - # Create Days 6-10 average files print("\nCreating Days 6-10 average files") valid_hr = int(valid_hr_start) @@ -83,29 +80,38 @@ days_avg_day_start = 10 days_avg_day = days_avg_day_start while days_avg_day <= days_avg_day_end: + full_path_job_num_work_dir = os.path.join( + job_num_work_dir, RUN+'.'+DATE, MODEL, VERIF_CASE + ) + full_path_DATA = os.path.join( + DATA, VERIF_CASE+'_'+STEP, 'METplus_output', + RUN+'.'+DATE, MODEL, VERIF_CASE + ) + full_path_COMOUT = os.path.join( + COMOUT, RUN+'.'+DATE, MODEL, VERIF_CASE + ) days_avg_file_list = [] days_avg_day_fhr_end = days_avg_day * 24 days_avg_day_fhr_start = days_avg_day_fhr_end - 120 days_avg_day_init = (days_avg_valid_end - datetime.timedelta(days=days_avg_day)) days_avg_day_fhr = days_avg_day_fhr_start - output_DATA_file = os.path.join(output_dir, 'days6_10_avg_' - +VERIF_TYPE+'_'+job_name+'_init' - +days_avg_day_init.strftime('%Y%m%d%H') - +'_valid' - +days_avg_valid_start\ - .strftime('%Y%m%d%H')+'to' - +days_avg_valid_end\ - .strftime('%Y%m%d%H')+'.stat') - output_COMOUT_file = os.path.join(COMOUT, RUN+'.'+DATE, MODEL, - VERIF_CASE, 'days6_10_avg_' - +VERIF_TYPE+'_'+job_name+'_init' - +days_avg_day_init.strftime('%Y%m%d%H') - +'_valid' - +days_avg_valid_start\ - .strftime('%Y%m%d%H')+'to' - +days_avg_valid_end\ - .strftime('%Y%m%d%H')+'.stat') + # Set output file + output_file = os.path.join(full_path_job_num_work_dir, + 'days6_10_avg_' + +VERIF_TYPE+'_'+job_name+'_init' + +days_avg_day_init.strftime('%Y%m%d%H') + +'_valid' + +days_avg_valid_start\ + .strftime('%Y%m%d%H')+'to' + +days_avg_valid_end\ + .strftime('%Y%m%d%H')+'.stat') + output_DATA_file = os.path.join( + full_path_DATA, output_file.rpartition('/')[2] + ) + output_COMOUT_file = os.path.join( + full_path_COMOUT, output_file.rpartition('/')[2] + ) while days_avg_day_fhr <= days_avg_day_fhr_end: days_avg_day_fhr_valid = ( days_avg_day_init @@ -147,23 +153,14 @@ make_days_avg_output_file = False else: if len(days_avg_file_list) >= 9: - if not os.path.exists(output_DATA_file): - make_days_avg_output_file = True - else: - make_days_avg_output_file = False - print(f"DATA Output File exist: {output_DATA_file}") - if SENDCOM == 'YES' \ - and sub_util.check_file_exists_size( - output_DATA_file - ): - sub_util.copy_file(output_DATA_file, - output_COMOUT_file) + make_days_avg_output_file = True else: print("WARNING: Need at least 9 files to create Days 6-10 average") make_days_avg_output_file = False if make_days_avg_output_file: - print(f"DATA Output File: {output_DATA_file}") - print(f"COMOUT Output File: {output_COMOUT_file}") + print(f"Output File: {output_file}") + if not os.path.exists(full_path_job_num_work_dir): + os.makedirs(full_path_job_num_work_dir) all_days_avg_df = pd.DataFrame(columns=MET_MPR_column_list) for days_avg_file in days_avg_file_list: with open(days_avg_file, 'r') as infile: @@ -258,12 +255,12 @@ days_avg_df_list, axis=1, ignore_index=True ).T days_avg_df.to_csv( - output_DATA_file, header=input_file_header, + output_file, header=input_file_header, index=None, sep=' ', mode='w' ) if SENDCOM == 'YES' \ - and sub_util.check_file_exists_size(output_DATA_file): - sub_util.copy_file(output_DATA_file, output_COMOUT_file) + and sub_util.check_file_exists_size(output_file): + sub_util.copy_file(output_file, output_COMOUT_file) print("") days_avg_day+=1 valid_hr+=int(valid_hr_inc) diff --git a/ush/subseasonal/subseasonal_stats_grid2obs_create_days6_10_reformat_job_scripts.py b/ush/subseasonal/subseasonal_stats_grid2obs_create_days6_10_reformat_job_scripts.py index 491b56b22..494862ff3 100644 --- a/ush/subseasonal/subseasonal_stats_grid2obs_create_days6_10_reformat_job_scripts.py +++ b/ush/subseasonal/subseasonal_stats_grid2obs_create_days6_10_reformat_job_scripts.py @@ -213,6 +213,7 @@ for model_idx in range(len(model_list)): job_env_dict['MODEL'] = model_list[model_idx] njobs = (int(njobs) + 1) + job_env_dict['job_num'] = str(njobs) # Create job file job_file = os.path.join(JOB_GROUP_jobs_dir, 'job'+str(njobs)) print("Creating job script: "+job_file) @@ -220,6 +221,15 @@ job.write('#!/bin/bash\n') job.write('set -x\n') job.write('\n') + # Create job working directory + job_env_dict['job_num_work_dir'] = os.path.join( + DATA, f"{VERIF_CASE}_{STEP}", 'METplus_output', + 'job_work_dir', JOB_GROUP, + f"job{job_env_dict['job_num']}" + ) + job_env_dict['MET_TMP_DIR'] = os.path.join( + job_env_dict['job_num_work_dir'], 'tmp' + ) # Set any environment variables for special cases if JOB_GROUP == 'reformat_data': if verif_type == 'prepbufr': @@ -248,7 +258,7 @@ check_truth_files = True if check_model_files: (model_files_exist, valid_date_fhr_list, - model_copy_output_DATA2COMOUT_list) = ( + model_copy_output_list) = ( sub_util.check_days6_10_model_files(job_env_dict) ) job_env_dict['fhr_list'] = ( @@ -259,7 +269,7 @@ job_env_dict.pop('fhr_inc') if check_truth_files: (all_truth_file_exist, - truth_copy_output_DATA2COMOUT_list) = ( + truth_copy_output_list) = ( sub_util.check_days6_10_truth_files(job_env_dict) ) if model_files_exist and all_truth_file_exist: @@ -277,14 +287,16 @@ job.write('\n') # Write job commands if write_job_cmds: + if not os.path.exists(job_env_dict['job_num_work_dir']): + os.makedirs(job_env_dict['job_num_work_dir']) for cmd in verif_type_job_commands_list: job.write(cmd+'\n') job.write('export err=$?; err_chk'+'\n') - # Copy DATA files to COMOUT restart dir + # Copy files to COMOUT restart dir # to be used in possible restart if job_env_dict['SENDCOM'] == 'YES': for model_output_file_tuple \ - in model_copy_output_DATA2COMOUT_list: + in model_copy_output_list: job.write(f'if [ -f "{model_output_file_tuple[0]}" ]; then ' +f"cp -v {model_output_file_tuple[0]} " +f"{model_output_file_tuple[1]}" diff --git a/ush/subseasonal/subseasonal_stats_grid2obs_create_job_scripts.py b/ush/subseasonal/subseasonal_stats_grid2obs_create_job_scripts.py index 024320019..bacb9e311 100644 --- a/ush/subseasonal/subseasonal_stats_grid2obs_create_job_scripts.py +++ b/ush/subseasonal/subseasonal_stats_grid2obs_create_job_scripts.py @@ -95,9 +95,7 @@ ), 'nweekly_avg_stat_files=' +'$(ls '+os.path.join( - '$DATA', - '${VERIF_CASE}_${STEP}', - 'METplus_output', + '$COMOUT', '${RUN}.${DATE}', '$MODEL', '$VERIF_CASE', 'weekly_avg_prepbufr_' @@ -151,9 +149,7 @@ ), 'ndays6_10_avg_stat_files=' +'$(ls '+os.path.join( - '$DATA', - '${VERIF_CASE}_${STEP}', - 'METplus_output', + '$COMOUT', '${RUN}.${DATE}', '$MODEL', '$VERIF_CASE', 'days6_10_avg_prepbufr_' @@ -207,9 +203,7 @@ ), 'nweeks3_4_avg_stat_files=' +'$(ls '+os.path.join( - '$DATA', - '${VERIF_CASE}_${STEP}', - 'METplus_output', + '$COMOUT', '${RUN}.${DATE}', '$MODEL', '$VERIF_CASE', 'weeks3_4_avg_prepbufr_' @@ -269,9 +263,7 @@ ), 'nweekly_avg_stat_files=' +'$(ls '+os.path.join( - '$DATA', - '${VERIF_CASE}_${STEP}', - 'METplus_output', + '$COMOUT', '${RUN}.${DATE}', '$MODEL', '$VERIF_CASE', 'weekly_avg_prepbufr_' @@ -332,9 +324,7 @@ ), 'ndays6_10_avg_stat_files=' +'$(ls '+os.path.join( - '$DATA', - '${VERIF_CASE}_${STEP}', - 'METplus_output', + '$COMOUT', '${RUN}.${DATE}', '$MODEL', '$VERIF_CASE', 'days6_10_avg_prepbufr_' @@ -395,9 +385,7 @@ ), 'nweeks3_4_avg_stat_files=' +'$(ls '+os.path.join( - '$DATA', - '${VERIF_CASE}_${STEP}', - 'METplus_output', + '$COMOUT', '${RUN}.${DATE}', '$MODEL', '$VERIF_CASE', 'weeks3_4_avg_prepbufr_' @@ -482,6 +470,7 @@ for model_idx in range(len(model_list)): job_env_dict['MODEL'] = model_list[model_idx] njobs+=1 + job_env_dict['job_num'] = str(njobs) # Create job file job_file = os.path.join(JOB_GROUP_jobs_dir, 'job'+str(njobs)) print("Creating job script: "+job_file) @@ -489,6 +478,15 @@ job.write('#!/bin/bash\n') job.write('set -x\n') job.write('\n') + # Create job working directory + job_env_dict['job_num_work_dir'] = os.path.join( + DATA, f"{VERIF_CASE}_{STEP}", 'METplus_output', + 'job_work_dir', JOB_GROUP, + f"job{job_env_dict['job_num']}" + ) + job_env_dict['MET_TMP_DIR'] = os.path.join( + job_env_dict['job_num_work_dir'], 'tmp' + ) # Set any environment variables for special cases if JOB_GROUP in ['assemble_data', 'generate_stats']: if verif_type == 'prepbufr': @@ -516,7 +514,7 @@ check_model_files = True if check_model_files: (model_files_exist, valid_date_fhr_list, - model_copy_output_DATA2COMOUT_list) = ( + model_copy_output_list) = ( sub_util.check_model_files(job_env_dict) ) job_env_dict['fhr_list'] = ( @@ -546,13 +544,15 @@ job.write('\n') # Write job commands if write_job_cmds: + if not os.path.exists(job_env_dict['job_num_work_dir']): + os.makedirs(job_env_dict['job_num_work_dir']) for cmd in verif_type_job_commands_list: job.write(cmd+'\n') job.write('export err=$?; err_chk'+'\n') - # Copy DATA files to COMOUT restart dir + # Copy files to COMOUT restart dir if job_env_dict['SENDCOM'] == 'YES': for model_output_file_tuple \ - in model_copy_output_DATA2COMOUT_list: + in model_copy_output_list: job.write(f'if [ -f "{model_output_file_tuple[0]}" ]; then ' +f"cp -v {model_output_file_tuple[0]} " +f"{model_output_file_tuple[1]}" @@ -649,6 +649,7 @@ for model_idx in range(len(model_list)): job_env_dict['MODEL'] = model_list[model_idx] njobs+=1 + job_env_dict['job_num'] = str(njobs) # Create job file job_file = os.path.join(JOB_GROUP_jobs_dir, 'job'+str(njobs)) print("Creating job script: "+job_file) @@ -656,6 +657,15 @@ job.write('#!/bin/bash\n') job.write('set -x\n') job.write('\n') + # Create job working directory + job_env_dict['job_num_work_dir'] = os.path.join( + DATA, f"{VERIF_CASE}_{STEP}", 'METplus_output', + 'job_work_dir', JOB_GROUP, + f"job{job_env_dict['job_num']}" + ) + job_env_dict['MET_TMP_DIR'] = os.path.join( + job_env_dict['job_num_work_dir'], 'tmp' + ) # Set any environment variables for special cases # Write environment variables for name, value in job_env_dict.items(): @@ -669,6 +679,8 @@ write_job_cmds = False # Write job commands if write_job_cmds: + if not os.path.exists(job_env_dict['job_num_work_dir']): + os.makedirs(job_env_dict['job_num_work_dir']) for cmd in gather_stats_jobs_dict['commands']: job.write(cmd+'\n') job.write('export err=$?; err_chk'+'\n') diff --git a/ush/subseasonal/subseasonal_stats_grid2obs_create_weekly_anomaly.py b/ush/subseasonal/subseasonal_stats_grid2obs_create_weekly_anomaly.py index 666c938b7..8b9ed57d4 100644 --- a/ush/subseasonal/subseasonal_stats_grid2obs_create_weekly_anomaly.py +++ b/ush/subseasonal/subseasonal_stats_grid2obs_create_weekly_anomaly.py @@ -32,28 +32,18 @@ MODEL = os.environ['MODEL'] WEEKLYSTART = os.environ['WEEKLYSTART'] DATE = os.environ['DATE'] +var1_obs_name = os.environ['var1_obs_name'] +var1_obs_levels = os.environ['var1_obs_levels'] valid_hr_start = os.environ['valid_hr_start'] valid_hr_end = os.environ['valid_hr_end'] valid_hr_inc = os.environ['valid_hr_inc'] fhr_list = os.environ['CORRECT_LEAD_SEQ'].split(',') +job_num_work_dir = os.environ['job_num_work_dir'] -# Process run time arguments -if len(sys.argv) != 3: - print("FATAL ERROR: Not given correct number of run time arguments..." - +os.path.basename(__file__)+" VARNAME_VARLEVEL FILE_FORMAT") +# Check variable settings +if ' ' in var1_obs_levels or ',' in var1_obs_levels: + print("ERROR: Cannot accept list of observation levels") sys.exit(1) -else: - if '_' not in sys.argv[1]: - print("FATAL ERROR: variable and level runtime argument formatted " - +"incorrectly, be sure to separate variable and level with " - +"an underscore (_), example TMP_Z2") - sys.exit(1) - else: - var_level = sys.argv[1] - print("Using var_level = "+var_level) - file_format = sys.argv[2] -var = var_level.split('_')[0] -level = var_level.split('_')[1] # Set MET MPR columns MET_MPR_column_list = [ @@ -78,46 +68,61 @@ valid_date_dt = STARTDATE_dt fhr = fhr_start while valid_date_dt <= ENDDATE_dt and fhr <= fhr_end: + # Set full paths for dates + full_path_job_num_work_dir = os.path.join( + job_num_work_dir, RUN+'.' + +ENDDATE_dt.strftime('%Y%m%d'), + MODEL, VERIF_CASE + ) + full_path_DATA = os.path.join( + DATA, VERIF_CASE+'_'+STEP, 'METplus_output', + RUN+'.'+ENDDATE_dt.strftime('%Y%m%d'), + MODEL, VERIF_CASE + ) + full_path_COMOUT = os.path.join( + COMOUT, RUN+'.'+ENDDATE_dt.strftime('%Y%m%d'), + MODEL, VERIF_CASE + ) init_date_dt = valid_date_dt - datetime.timedelta(hours=fhr) - input_file = sub_util.format_filler( - file_format, valid_date_dt, init_date_dt, str(fhr), {} + input_file_name = ( + f"point_stat_{VERIF_TYPE}_{job_name}_{str(fhr).zfill(2)}0000L_" + +f"{valid_date_dt:%Y%m%d_%H}0000V.stat" ) - if os.path.exists(input_file): - output_dir = os.path.join(DATA, VERIF_CASE+'_'+STEP, - 'METplus_output', - RUN+'.' - +ENDDATE_dt.strftime('%Y%m%d'), - MODEL, VERIF_CASE) - output_DATA_file = os.path.join(output_dir, 'anomaly_' - +VERIF_TYPE+'_'+job_name+'_init' - +init_date_dt.strftime('%Y%m%d%H')+'_' - +'fhr'+str(fhr).zfill(3)+'.stat') - output_COMOUT_file = os.path.join(COMOUT, RUN+'.' - +ENDDATE_dt.strftime('%Y%m%d'), - MODEL, VERIF_CASE, 'anomaly_' - +VERIF_TYPE+'_'+job_name+'_init' - +init_date_dt.strftime('%Y%m%d%H')+'_' - +'fhr'+str(fhr).zfill(3)+'.stat') - if os.path.exists(output_COMOUT_file): - make_anomaly_output_file = False - sub_util.copy_file(output_COMOUT_file, output_DATA_file) - else: - if not os.path.exists(output_DATA_file): - make_anomaly_output_file = True - else: - make_anomaly_output_file = False - print(f"DATA Output File exists: {output_DATA_file}") - if SENDCOM == 'YES' \ - and sub_util.check_file_size_exists( - output_DATA_file - ): - sub_util.copy_file(output_DATA_file, - output_COMOUT_file) - else: - print(f"\nWARNING: {input_file} does not exist") + # Check possible input files + check_input_file_list = [ + os.path.join(full_path_job_num_work_dir, input_file_name), + os.path.join(full_path_DATA, input_file_name), + os.path.join(full_path_COMOUT, input_file_name) + ] + found_input = False + for check_input_file in check_input_file_list: + if os.path.exists(check_input_file): + input_file = check_input_file + found_input = True + break + # Set output file + output_file = os.path.join( + full_path_job_num_work_dir, f"anomaly_{VERIF_TYPE}_{job_name}_" + +f"init{init_date_dt:%Y%m%d%H}_fhr{str(fhr).zfill(3)}.stat" + ) + output_DATA_file = os.path.join( + full_path_DATA, output_file.rpartition('/')[2] + ) + output_COMOUT_file = os.path.join( + full_path_COMOUT, output_file.rpartition('/')[2] + ) + # Check input and output files + if os.path.exists(output_COMOUT_file): + print(f"COMOUT Output File exists: {output_COMOUT_file}") make_anomaly_output_file = False - if make_anomaly_output_file: + sub_util.copy_file(output_COMOUT_file, output_DATA_file) + else: + make_anomaly_output_file = True + if found_input and make_anomaly_output_file: print("\nInput file: "+input_file) + print(f"Output File: {output_file}") + if not os.path.exists(full_path_job_num_work_dir): + os.makedirs(full_path_job_num_work_dir) with open(input_file, 'r') as infile: input_file_header = infile.readline() sub_util.run_shell_command(['sed', '-i', '"s/ a//g"', @@ -127,7 +132,10 @@ names=MET_MPR_column_list, na_filter=False, dtype=str) input_file_var_level_df = input_file_df[ - (input_file_df['FCST_VAR'] == var) & (input_file_df['FCST_LEV'] == level) + (input_file_df['FCST_VAR'] == var1_obs_name) \ + & (input_file_df['FCST_LEV'] == var1_obs_levels) \ + & (input_file_df['OBS_VAR'] == var1_obs_name) \ + & (input_file_df['OBS_LEV'] == var1_obs_levels) ] fcst_var_level = np.array( input_file_var_level_df['FCST'].values, dtype=float @@ -145,15 +153,13 @@ output_file_df['CLIMO_MEAN'] = 'NA' output_file_df['FCST'] = fcst_anom_var_level output_file_df['OBS'] = obs_anom_var_level - output_file_df['FCST_VAR'] = var+'_ANOM' - output_file_df['OBS_VAR'] = var+'_ANOM' - print(f"DATA Output File: {output_DATA_file}") - print(f"COMOUT Output File: {output_COMOUT_file}") - output_file_df.to_csv(output_DATA_file, header=input_file_header, + output_file_df['FCST_VAR'] = f"{var1_obs_name}_ANOM" + output_file_df['OBS_VAR'] = f"{var1_obs_name}_ANOM" + output_file_df.to_csv(output_file, header=input_file_header, index=None, sep=' ', mode='w') if SENDCOM == 'YES' \ - and sub_util.check_file_exists_size(output_DATA_file): - sub_util.copy_file(output_DATA_file, output_COMOUT_file) + and sub_util.check_file_exists_size(output_file): + sub_util.copy_file(output_file, output_COMOUT_file) valid_date_dt = valid_date_dt + datetime.timedelta(hours=int(valid_hr_inc)) fhr+=int(valid_hr_inc) diff --git a/ush/subseasonal/subseasonal_stats_grid2obs_create_weekly_assemble_job_scripts.py b/ush/subseasonal/subseasonal_stats_grid2obs_create_weekly_assemble_job_scripts.py index 010ebf482..043c3bf8b 100644 --- a/ush/subseasonal/subseasonal_stats_grid2obs_create_weekly_assemble_job_scripts.py +++ b/ush/subseasonal/subseasonal_stats_grid2obs_create_weekly_assemble_job_scripts.py @@ -83,22 +83,7 @@ 'subseasonal_stats_' 'grid2obs_create_' 'weekly_anomaly.py', - ['TMP_Z2', - os.path.join( - '$DATA', - '${VERIF_CASE}_${STEP}', - 'METplus_output', - '${RUN}.' - +'$DATE', - '$MODEL', '$VERIF_CASE', - 'point_stat_' - +'${VERIF_TYPE}_' - +'${job_name}_' - +'{lead?fmt=%2H}0000L_' - +'{valid?fmt=%Y%m%d}_' - +'{valid?fmt=%H}0000V' - +'.stat' - )] + [] )]}, } } @@ -245,6 +230,7 @@ for model_idx in range(len(model_list)): job_env_dict['MODEL'] = model_list[model_idx] njobs = (int(njobs) + 1) + job_env_dict['job_num'] = str(njobs) # Create job file job_file = os.path.join(JOB_GROUP_jobs_dir, 'job'+str(njobs)) print("Creating job script: "+job_file) @@ -252,6 +238,15 @@ job.write('#!/bin/bash\n') job.write('set -x\n') job.write('\n') + # Create job working directory + job_env_dict['job_num_work_dir'] = os.path.join( + DATA, f"{VERIF_CASE}_{STEP}", 'METplus_output', + 'job_work_dir', JOB_GROUP, + f"job{job_env_dict['job_num']}" + ) + job_env_dict['MET_TMP_DIR'] = os.path.join( + job_env_dict['job_num_work_dir'], 'tmp' + ) # Set any environment variables for special cases if JOB_GROUP == 'assemble_data': if verif_type == 'prepbufr': @@ -280,7 +275,7 @@ check_truth_files = True if check_model_files: (model_files_exist, valid_date_fhr_list, - model_copy_output_DATA2COMOUT_list) = ( + model_copy_output_list) = ( sub_util.check_weekly_model_files(job_env_dict) ) job_env_dict['fhr_list'] = ( @@ -291,7 +286,7 @@ job_env_dict.pop('fhr_inc') if check_truth_files: (all_truth_file_exist, - truth_copy_output_DATA2COMOUT_list) = ( + truth_copy_output_list) = ( sub_util.check_weekly_truth_files(job_env_dict) ) if model_files_exist and all_truth_file_exist: @@ -309,14 +304,16 @@ job.write('\n') # Write job commands if write_job_cmds: + if not os.path.exists(job_env_dict['job_num_work_dir']): + os.makedirs(job_env_dict['job_num_work_dir']) for cmd in verif_type_job_commands_list: job.write(cmd+'\n') job.write('export err=$?; err_chk'+'\n') - # Copy DATA files to COMOUT restart dir + # Copy files to COMOUT restart dir # to be used in possible restart if job_env_dict['SENDCOM'] == 'YES': for model_output_file_tuple \ - in model_copy_output_DATA2COMOUT_list: + in model_copy_output_list: job.write(f'if [ -f "{model_output_file_tuple[0]}" ]; then ' +f"cp -v {model_output_file_tuple[0]} " +f"{model_output_file_tuple[1]}" diff --git a/ush/subseasonal/subseasonal_stats_grid2obs_create_weekly_avg.py b/ush/subseasonal/subseasonal_stats_grid2obs_create_weekly_avg.py index 9d5f34863..e27ab151d 100644 --- a/ush/subseasonal/subseasonal_stats_grid2obs_create_weekly_avg.py +++ b/ush/subseasonal/subseasonal_stats_grid2obs_create_weekly_avg.py @@ -36,6 +36,7 @@ valid_hr_end = os.environ['valid_hr_end'] valid_hr_inc = os.environ['valid_hr_inc'] fhr_end = os.environ['fhr_list'].split(',')[-1] +job_num_work_dir = os.environ['job_num_work_dir'] # Process run time arguments if len(sys.argv) != 4: @@ -66,10 +67,6 @@ 'CLIMO_MEAN', 'CLIMO_STDEV', 'CLIMO_CDF' ] -# Set input and output directories -output_dir = os.path.join(DATA, VERIF_CASE+'_'+STEP, 'METplus_output', - RUN+'.'+DATE, MODEL, VERIF_CASE) - # Create weekly average files print("\nCreating weekly average files") valid_hr = int(valid_hr_start) @@ -83,29 +80,38 @@ weekly_avg_day_start = 7 weekly_avg_day = weekly_avg_day_start while weekly_avg_day <= weekly_avg_day_end: + full_path_job_num_work_dir = os.path.join( + job_num_work_dir, RUN+'.'+DATE, MODEL, VERIF_CASE + ) + full_path_DATA = os.path.join( + DATA, VERIF_CASE+'_'+STEP, 'METplus_output', + RUN+'.'+DATE, MODEL, VERIF_CASE + ) + full_path_COMOUT = os.path.join( + COMOUT, RUN+'.'+DATE, MODEL, VERIF_CASE + ) weekly_avg_file_list = [] weekly_avg_day_fhr_end = weekly_avg_day * 24 weekly_avg_day_fhr_start = weekly_avg_day_fhr_end - 168 weekly_avg_day_init = (weekly_avg_valid_end - datetime.timedelta(days=weekly_avg_day)) weekly_avg_day_fhr = weekly_avg_day_fhr_start - output_DATA_file = os.path.join(output_dir, 'weekly_avg_' - +VERIF_TYPE+'_'+job_name+'_init' - +weekly_avg_day_init.strftime('%Y%m%d%H') - +'_valid' - +weekly_avg_valid_start\ - .strftime('%Y%m%d%H')+'to' - +weekly_avg_valid_end\ - .strftime('%Y%m%d%H')+'.stat') - output_COMOUT_file = os.path.join(COMOUT, RUN+'.'+DATE, MODEL, - VERIF_CASE, 'weekly_avg_' - +VERIF_TYPE+'_'+job_name+'_init' - +weekly_avg_day_init.strftime('%Y%m%d%H') - +'_valid' - +weekly_avg_valid_start\ - .strftime('%Y%m%d%H')+'to' - +weekly_avg_valid_end\ - .strftime('%Y%m%d%H')+'.stat') + # Set output file + output_file = os.path.join(full_path_job_num_work_dir, + 'weekly_avg_' + +VERIF_TYPE+'_'+job_name+'_init' + +weekly_avg_day_init.strftime('%Y%m%d%H') + +'_valid' + +weekly_avg_valid_start\ + .strftime('%Y%m%d%H')+'to' + +weekly_avg_valid_end\ + .strftime('%Y%m%d%H')+'.stat') + output_DATA_file = os.path.join( + full_path_DATA, output_file.rpartition('/')[2] + ) + output_COMOUT_file = os.path.join( + full_path_COMOUT, output_file.rpartition('/')[2] + ) while weekly_avg_day_fhr <= weekly_avg_day_fhr_end: weekly_avg_day_fhr_valid = ( weekly_avg_day_init @@ -147,23 +153,14 @@ make_weekly_avg_output_file = False else: if len(weekly_avg_file_list) >= 12: - if not os.path.exists(output_DATA_file): - make_weekly_avg_output_file = True - else: - make_weekly_avg_output_file = False - print(f"DATA Output File exist: {output_DATA_file}") - if SENDCOM == 'YES' \ - and sub_util.check_file_exists_size( - output_DATA_file - ): - sub_util.copy_file(output_DATA_file, - output_COMOUT_file) + make_weekly_avg_output_file = True else: print("WARNING: Need at least 12 files to create weekly average") make_weekly_avg_output_file = False if make_weekly_avg_output_file: - print(f"DATA Output File: {output_DATA_file}") - print(f"COMOUT Output File: {output_COMOUT_file}") + print(f"Output File: {output_file}") + if not os.path.exists(full_path_job_num_work_dir): + os.makedirs(full_path_job_num_work_dir) all_weekly_avg_df = pd.DataFrame(columns=MET_MPR_column_list) for weekly_avg_file in weekly_avg_file_list: with open(weekly_avg_file, 'r') as infile: @@ -258,12 +255,12 @@ weekly_avg_df_list, axis=1, ignore_index=True ).T weekly_avg_df.to_csv( - output_DATA_file, header=input_file_header, + output_file, header=input_file_header, index=None, sep=' ', mode='w' ) if SENDCOM == 'YES' \ - and sub_util.check_file_exists_size(output_DATA_file): - sub_util.copy_file(output_DATA_file, output_COMOUT_file) + and sub_util.check_file_exists_size(output_file): + sub_util.copy_file(output_file, output_COMOUT_file) print("") weekly_avg_day+=7 valid_hr+=int(valid_hr_inc) diff --git a/ush/subseasonal/subseasonal_stats_grid2obs_create_weekly_reformat_job_scripts.py b/ush/subseasonal/subseasonal_stats_grid2obs_create_weekly_reformat_job_scripts.py index df550a1d4..eac239222 100644 --- a/ush/subseasonal/subseasonal_stats_grid2obs_create_weekly_reformat_job_scripts.py +++ b/ush/subseasonal/subseasonal_stats_grid2obs_create_weekly_reformat_job_scripts.py @@ -213,6 +213,7 @@ for model_idx in range(len(model_list)): job_env_dict['MODEL'] = model_list[model_idx] njobs = (int(njobs) + 1) + job_env_dict['job_num'] = str(njobs) # Create job file job_file = os.path.join(JOB_GROUP_jobs_dir, 'job'+str(njobs)) print("Creating job script: "+job_file) @@ -220,6 +221,15 @@ job.write('#!/bin/bash\n') job.write('set -x\n') job.write('\n') + # Create job working directory + job_env_dict['job_num_work_dir'] = os.path.join( + DATA, f"{VERIF_CASE}_{STEP}", 'METplus_output', + 'job_work_dir', JOB_GROUP, + f"job{job_env_dict['job_num']}" + ) + job_env_dict['MET_TMP_DIR'] = os.path.join( + job_env_dict['job_num_work_dir'], 'tmp' + ) # Set any environment variables for special cases if JOB_GROUP == 'reformat_data': if verif_type == 'prepbufr': @@ -248,7 +258,7 @@ check_truth_files = True if check_model_files: (model_files_exist, valid_date_fhr_list, - model_copy_output_DATA2COMOUT_list) = ( + model_copy_output_list) = ( sub_util.check_weekly_model_files(job_env_dict) ) job_env_dict['fhr_list'] = ( @@ -259,7 +269,7 @@ job_env_dict.pop('fhr_inc') if check_truth_files: (all_truth_file_exist, - truth_copy_output_DATA2COMOUT_list) = ( + truth_copy_output_list) = ( sub_util.check_weekly_truth_files(job_env_dict) ) if model_files_exist and all_truth_file_exist: @@ -277,14 +287,16 @@ job.write('\n') # Write job commands if write_job_cmds: + if not os.path.exists(job_env_dict['job_num_work_dir']): + os.makedirs(job_env_dict['job_num_work_dir']) for cmd in verif_type_job_commands_list: job.write(cmd+'\n') job.write('export err=$?; err_chk'+'\n') - # Copy DATA files to COMOUT restart dir + # Copy files to COMOUT restart dir # to be used in possible restart if job_env_dict['SENDCOM'] == 'YES': for model_output_file_tuple \ - in model_copy_output_DATA2COMOUT_list: + in model_copy_output_list: job.write(f'if [ -f "{model_output_file_tuple[0]}" ]; then ' +f"cp -v {model_output_file_tuple[0]} " +f"{model_output_file_tuple[1]}" diff --git a/ush/subseasonal/subseasonal_stats_grid2obs_create_weeks3_4_anomaly.py b/ush/subseasonal/subseasonal_stats_grid2obs_create_weeks3_4_anomaly.py index 1819f7c03..0a526b322 100644 --- a/ush/subseasonal/subseasonal_stats_grid2obs_create_weeks3_4_anomaly.py +++ b/ush/subseasonal/subseasonal_stats_grid2obs_create_weeks3_4_anomaly.py @@ -32,28 +32,18 @@ MODEL = os.environ['MODEL'] W3_4START = os.environ['W3_4START'] DATE = os.environ['DATE'] +var1_obs_name = os.environ['var1_obs_name'] +var1_obs_levels = os.environ['var1_obs_levels'] valid_hr_start = os.environ['valid_hr_start'] valid_hr_end = os.environ['valid_hr_end'] valid_hr_inc = os.environ['valid_hr_inc'] fhr_list = os.environ['CORRECT_LEAD_SEQ'].split(',') +job_num_work_dir = os.environ['job_num_work_dir'] -# Process run time arguments -if len(sys.argv) != 3: - print("FATAL ERROR: Not given correct number of run time arguments..." - +os.path.basename(__file__)+" VARNAME_VARLEVEL FILE_FORMAT") +# Check variable settings +if ' ' in var1_obs_levels or ',' in var1_obs_levels: + print("ERROR: Cannot accept list of observation levels") sys.exit(1) -else: - if '_' not in sys.argv[1]: - print("FATAL ERROR: variable and level runtime argument formatted " - +"incorrectly, be sure to separate variable and level with " - +"an underscore (_), example TMP_Z2") - sys.exit(1) - else: - var_level = sys.argv[1] - print("Using var_level = "+var_level) - file_format = sys.argv[2] -var = var_level.split('_')[0] -level = var_level.split('_')[1] # Set MET MPR columns MET_MPR_column_list = [ @@ -78,46 +68,61 @@ valid_date_dt = STARTDATE_dt fhr = fhr_start while valid_date_dt <= ENDDATE_dt and fhr <= fhr_end: + # Set full paths for dates + full_path_job_num_work_dir = os.path.join( + job_num_work_dir, RUN+'.' + +ENDDATE_dt.strftime('%Y%m%d'), + MODEL, VERIF_CASE + ) + full_path_DATA = os.path.join( + DATA, VERIF_CASE+'_'+STEP, 'METplus_output', + RUN+'.'+ENDDATE_dt.strftime('%Y%m%d'), + MODEL, VERIF_CASE + ) + full_path_COMOUT = os.path.join( + COMOUT, RUN+'.'+ENDDATE_dt.strftime('%Y%m%d'), + MODEL, VERIF_CASE + ) init_date_dt = valid_date_dt - datetime.timedelta(hours=fhr) - input_file = sub_util.format_filler( - file_format, valid_date_dt, init_date_dt, str(fhr), {} + input_file_name = ( + f"point_stat_{VERIF_TYPE}_{job_name}_{str(fhr).zfill(2)}0000L_" + +f"{valid_date_dt:%Y%m%d_%H}0000V.stat" ) - if os.path.exists(input_file): - output_dir = os.path.join(DATA, VERIF_CASE+'_'+STEP, - 'METplus_output', - RUN+'.' - +ENDDATE_dt.strftime('%Y%m%d'), - MODEL, VERIF_CASE) - output_DATA_file = os.path.join(output_dir, 'anomaly_' - +VERIF_TYPE+'_'+job_name+'_init' - +init_date_dt.strftime('%Y%m%d%H')+'_' - +'fhr'+str(fhr).zfill(3)+'.stat') - output_COMOUT_file = os.path.join(COMOUT, RUN+'.' - +ENDDATE_dt.strftime('%Y%m%d'), - MODEL, VERIF_CASE, 'anomaly_' - +VERIF_TYPE+'_'+job_name+'_init' - +init_date_dt.strftime('%Y%m%d%H')+'_' - +'fhr'+str(fhr).zfill(3)+'.stat') - if os.path.exists(output_COMOUT_file): - make_anomaly_output_file = False - sub_util.copy_file(output_COMOUT_file, output_DATA_file) - else: - if not os.path.exists(output_DATA_file): - make_anomaly_output_file = True - else: - make_anomaly_output_file = False - print(f"DATA Output File exists: {output_DATA_file}") - if SENDCOM == 'YES' \ - and sub_util.check_file_size_exists( - output_DATA_file - ): - sub_util.copy_file(output_DATA_file, - output_COMOUT_file) - else: - print(f"\nWARNING: {input_file} does not exist") + # Check possible input files + check_input_file_list = [ + os.path.join(full_path_job_num_work_dir, input_file_name), + os.path.join(full_path_DATA, input_file_name), + os.path.join(full_path_COMOUT, input_file_name) + ] + found_input = False + for check_input_file in check_input_file_list: + if os.path.exists(check_input_file): + input_file = check_input_file + found_input = True + break + # Set output file + output_file = os.path.join( + full_path_job_num_work_dir, f"anomaly_{VERIF_TYPE}_{job_name}_" + +f"init{init_date_dt:%Y%m%d%H}_fhr{str(fhr).zfill(3)}.stat" + ) + output_DATA_file = os.path.join( + full_path_DATA, output_file.rpartition('/')[2] + ) + output_COMOUT_file = os.path.join( + full_path_COMOUT, output_file.rpartition('/')[2] + ) + # Check input and output files + if os.path.exists(output_COMOUT_file): + print(f"COMOUT Output File exists: {output_COMOUT_file}") make_anomaly_output_file = False - if make_anomaly_output_file: + sub_util.copy_file(output_COMOUT_file, output_DATA_file) + else: + make_anomaly_output_file = True + if found_input and make_anomaly_output_file: print("\nInput file: "+input_file) + print(f"Output File: {output_file}") + if not os.path.exists(full_path_job_num_work_dir): + os.makedirs(full_path_job_num_work_dir) with open(input_file, 'r') as infile: input_file_header = infile.readline() sub_util.run_shell_command(['sed', '-i', '"s/ a//g"', @@ -127,7 +132,10 @@ names=MET_MPR_column_list, na_filter=False, dtype=str) input_file_var_level_df = input_file_df[ - (input_file_df['FCST_VAR'] == var) & (input_file_df['FCST_LEV'] == level) + (input_file_df['FCST_VAR'] == var1_obs_name) \ + & (input_file_df['FCST_LEV'] == var1_obs_levels) \ + & (input_file_df['OBS_VAR'] == var1_obs_name) \ + & (input_file_df['OBS_LEV'] == var1_obs_levels) ] fcst_var_level = np.array( input_file_var_level_df['FCST'].values, dtype=float @@ -145,15 +153,13 @@ output_file_df['CLIMO_MEAN'] = 'NA' output_file_df['FCST'] = fcst_anom_var_level output_file_df['OBS'] = obs_anom_var_level - output_file_df['FCST_VAR'] = var+'_ANOM' - output_file_df['OBS_VAR'] = var+'_ANOM' - print(f"DATA Output File: {output_DATA_file}") - print(f"COMOUT Output File: {output_COMOUT_file}") - output_file_df.to_csv(output_DATA_file, header=input_file_header, + output_file_df['FCST_VAR'] = f"{var1_obs_name}_ANOM" + output_file_df['OBS_VAR'] = f"{var1_obs_name}_ANOM" + output_file_df.to_csv(output_file, header=input_file_header, index=None, sep=' ', mode='w') if SENDCOM == 'YES' \ - and sub_util.check_file_exists_size(output_DATA_file): - sub_util.copy_file(output_DATA_file, output_COMOUT_file) + and sub_util.check_file_exists_size(output_file): + sub_util.copy_file(output_file, output_COMOUT_file) valid_date_dt = valid_date_dt + datetime.timedelta(hours=int(valid_hr_inc)) fhr+=int(valid_hr_inc) diff --git a/ush/subseasonal/subseasonal_stats_grid2obs_create_weeks3_4_assemble_job_scripts.py b/ush/subseasonal/subseasonal_stats_grid2obs_create_weeks3_4_assemble_job_scripts.py index c21d57920..e616e6e68 100644 --- a/ush/subseasonal/subseasonal_stats_grid2obs_create_weeks3_4_assemble_job_scripts.py +++ b/ush/subseasonal/subseasonal_stats_grid2obs_create_weeks3_4_assemble_job_scripts.py @@ -83,22 +83,7 @@ 'subseasonal_stats_' 'grid2obs_create_' 'weeks3_4_anomaly.py', - ['TMP_Z2', - os.path.join( - '$DATA', - '${VERIF_CASE}_${STEP}', - 'METplus_output', - '${RUN}.' - +'$DATE', - '$MODEL', '$VERIF_CASE', - 'point_stat_' - +'${VERIF_TYPE}_' - +'${job_name}_' - +'{lead?fmt=%2H}0000L_' - +'{valid?fmt=%Y%m%d}_' - +'{valid?fmt=%H}0000V' - +'.stat' - )] + [] )]}, } } @@ -245,6 +230,7 @@ for model_idx in range(len(model_list)): job_env_dict['MODEL'] = model_list[model_idx] njobs = (int(njobs) + 1) + job_env_dict['job_num'] = str(njobs) # Create job file job_file = os.path.join(JOB_GROUP_jobs_dir, 'job'+str(njobs)) print("Creating job script: "+job_file) @@ -252,6 +238,15 @@ job.write('#!/bin/bash\n') job.write('set -x\n') job.write('\n') + # Create job working directory + job_env_dict['job_num_work_dir'] = os.path.join( + DATA, f"{VERIF_CASE}_{STEP}", 'METplus_output', + 'job_work_dir', JOB_GROUP, + f"job{job_env_dict['job_num']}" + ) + job_env_dict['MET_TMP_DIR'] = os.path.join( + job_env_dict['job_num_work_dir'], 'tmp' + ) # Set any environment variables for special cases if JOB_GROUP == 'assemble_data': if verif_type == 'prepbufr': @@ -280,7 +275,7 @@ check_truth_files = True if check_model_files: (model_files_exist, valid_date_fhr_list, - model_copy_output_DATA2COMOUT_list) = ( + model_copy_output_list) = ( sub_util.check_weeks3_4_model_files(job_env_dict) ) job_env_dict['fhr_list'] = ( @@ -291,7 +286,7 @@ job_env_dict.pop('fhr_inc') if check_truth_files: (all_truth_file_exist, - truth_copy_output_DATA2COMOUT_list) = ( + truth_copy_output_list) = ( sub_util.check_weeks3_4_truth_files(job_env_dict) ) if model_files_exist and all_truth_file_exist: @@ -309,14 +304,16 @@ job.write('\n') # Write job commands if write_job_cmds: + if not os.path.exists(job_env_dict['job_num_work_dir']): + os.makedirs(job_env_dict['job_num_work_dir']) for cmd in verif_type_job_commands_list: job.write(cmd+'\n') job.write('export err=$?; err_chk'+'\n') - # Copy DATA files to COMOUT restart dir + # Copy files to COMOUT restart dir # to be used in possible restart if job_env_dict['SENDCOM'] == 'YES': for model_output_file_tuple \ - in model_copy_output_DATA2COMOUT_list: + in model_copy_output_list: job.write(f'if [ -f "{model_output_file_tuple[0]}" ]; then ' +f"cp -v {model_output_file_tuple[0]} " +f"{model_output_file_tuple[1]}" diff --git a/ush/subseasonal/subseasonal_stats_grid2obs_create_weeks3_4_avg.py b/ush/subseasonal/subseasonal_stats_grid2obs_create_weeks3_4_avg.py index b77c4ad51..03f14cfc8 100644 --- a/ush/subseasonal/subseasonal_stats_grid2obs_create_weeks3_4_avg.py +++ b/ush/subseasonal/subseasonal_stats_grid2obs_create_weeks3_4_avg.py @@ -36,6 +36,7 @@ valid_hr_end = os.environ['valid_hr_end'] valid_hr_inc = os.environ['valid_hr_inc'] fhr_end = os.environ['fhr_list'].split(',')[-1] +job_num_work_dir = os.environ['job_num_work_dir'] # Process run time arguments if len(sys.argv) != 4: @@ -66,10 +67,6 @@ 'CLIMO_MEAN', 'CLIMO_STDEV', 'CLIMO_CDF' ] -# Set input and output directories -output_dir = os.path.join(DATA, VERIF_CASE+'_'+STEP, 'METplus_output', - RUN+'.'+DATE, MODEL, VERIF_CASE) - # Create Weeks 3-4 average files print("\nCreating Weeks 3-4 average files") valid_hr = int(valid_hr_start) @@ -83,29 +80,38 @@ weeks_avg_day_start = 28 weeks_avg_day = weeks_avg_day_start while weeks_avg_day <= weeks_avg_day_end: + full_path_job_num_work_dir = os.path.join( + job_num_work_dir, RUN+'.'+DATE, MODEL, VERIF_CASE + ) + full_path_DATA = os.path.join( + DATA, VERIF_CASE+'_'+STEP, 'METplus_output', + RUN+'.'+DATE, MODEL, VERIF_CASE + ) + full_path_COMOUT = os.path.join( + COMOUT, RUN+'.'+DATE, MODEL, VERIF_CASE + ) weeks_avg_file_list = [] weeks_avg_day_fhr_end = weeks_avg_day * 24 weeks_avg_day_fhr_start = weeks_avg_day_fhr_end - 336 weeks_avg_day_init = (weeks_avg_valid_end - datetime.timedelta(days=weeks_avg_day)) weeks_avg_day_fhr = weeks_avg_day_fhr_start - output_DATA_file = os.path.join(output_dir, 'weeks3_4_avg_' - +VERIF_TYPE+'_'+job_name+'_init' - +weeks_avg_day_init.strftime('%Y%m%d%H') - +'_valid' - +weeks_avg_valid_start\ - .strftime('%Y%m%d%H')+'to' - +weeks_avg_valid_end\ - .strftime('%Y%m%d%H')+'.stat') - output_COMOUT_file = os.path.join(COMOUT, RUN+'.'+DATE, MODEL, - VERIF_CASE, 'weeks3_4_avg_' - +VERIF_TYPE+'_'+job_name+'_init' - +weeks_avg_day_init.strftime('%Y%m%d%H') - +'_valid' - +weeks_avg_valid_start\ - .strftime('%Y%m%d%H')+'to' - +weeks_avg_valid_end\ - .strftime('%Y%m%d%H')+'.stat') + # Set output file + output_file = os.path.join(full_path_job_num_work_dir, + 'weeks3_4_avg_' + +VERIF_TYPE+'_'+job_name+'_init' + +weeks_avg_day_init.strftime('%Y%m%d%H') + +'_valid' + +weeks_avg_valid_start\ + .strftime('%Y%m%d%H')+'to' + +weeks_avg_valid_end\ + .strftime('%Y%m%d%H')+'.stat') + output_DATA_file = os.path.join( + full_path_DATA, output_file.rpartition('/')[2] + ) + output_COMOUT_file = os.path.join( + full_path_COMOUT, output_file.rpartition('/')[2] + ) while weeks_avg_day_fhr <= weeks_avg_day_fhr_end: weeks_avg_day_fhr_valid = ( weeks_avg_day_init @@ -147,23 +153,14 @@ make_weeks_avg_output_file = False else: if len(weeks_avg_file_list) >= 23: - if not os.path.exists(output_DATA_file): - make_weeks_avg_output_file = True - else: - make_weeks_avg_output_file = False - print(f"DATA Output File exist: {output_DATA_file}") - if SENDCOM == 'YES' \ - and sub_util.check_file_exists_size( - output_DATA_file - ): - sub_util.copy_file(output_DATA_file, - output_COMOUT_file) + make_weeks_avg_output_file = True else: print("WARNING: Need at least 23 files to create Weeks 3-4 average") make_weeks_avg_output_file = False if make_weeks_avg_output_file: - print(f"DATA Output File: {output_DATA_file}") - print(f"COMOUT Output File: {output_COMOUT_file}") + print(f"Output File: {output_file}") + if not os.path.exists(full_path_job_num_work_dir): + os.makedirs(full_path_job_num_work_dir) all_weeks_avg_df = pd.DataFrame(columns=MET_MPR_column_list) for weeks_avg_file in weeks_avg_file_list: with open(weeks_avg_file, 'r') as infile: @@ -258,12 +255,12 @@ weeks_avg_df_list, axis=1, ignore_index=True ).T weeks_avg_df.to_csv( - output_DATA_file, header=input_file_header, + output_file, header=input_file_header, index=None, sep=' ', mode='w' ) if SENDCOM == 'YES' \ - and sub_util.check_file_exists_size(output_DATA_file): - sub_util.copy_file(output_DATA_file, output_COMOUT_file) + and sub_util.check_file_exists_size(output_file): + sub_util.copy_file(output_file, output_COMOUT_file) print("") weeks_avg_day+=1 valid_hr+=int(valid_hr_inc) diff --git a/ush/subseasonal/subseasonal_stats_grid2obs_create_weeks3_4_reformat_job_scripts.py b/ush/subseasonal/subseasonal_stats_grid2obs_create_weeks3_4_reformat_job_scripts.py index 3d2395b22..01408bb7b 100644 --- a/ush/subseasonal/subseasonal_stats_grid2obs_create_weeks3_4_reformat_job_scripts.py +++ b/ush/subseasonal/subseasonal_stats_grid2obs_create_weeks3_4_reformat_job_scripts.py @@ -140,6 +140,7 @@ job_env_dict['valid_hr_start'] = date_dt.strftime('%H') job_env_dict['valid_hr_end'] = date_dt.strftime('%H') njobs = (int(njobs) + 1) + job_env_dict['job_num'] = str(njobs) # Create job file job_file = os.path.join(JOB_GROUP_jobs_dir, 'job'+str(njobs)) print("Creating job script: "+job_file) @@ -147,10 +148,19 @@ job.write('#!/bin/bash\n') job.write('set -x\n') job.write('\n') + # Create job working directory + job_env_dict['job_num_work_dir'] = os.path.join( + DATA, f"{VERIF_CASE}_{STEP}", 'METplus_output', + 'job_work_dir', JOB_GROUP, + f"job{job_env_dict['job_num']}" + ) + job_env_dict['MET_TMP_DIR'] = os.path.join( + job_env_dict['job_num_work_dir'], 'tmp' + ) # Set any environment variables for special cases # Do file checks (all_truth_file_exist, - truth_copy_output_DATA2COMOUT_list) = ( + truth_copy_output_list) = ( sub_util.check_weeks3_4_truth_files(job_env_dict) ) if all_truth_file_exist: @@ -163,13 +173,15 @@ job.write('\n') # Write job commands if write_job_cmds: + if not os.path.exists(job_env_dict['job_num_work_dir']): + os.makedirs(job_env_dict['job_num_work_dir']) for cmd in verif_type_job_commands_list: job.write(cmd+'\n') job.write('export err=$?; err_chk'+'\n') - # Copy DATA files to COMOUT restart dir + # Copy files to COMOUT restart dir if job_env_dict['SENDCOM'] == 'YES': for truth_output_file_tuple \ - in truth_copy_output_DATA2COMOUT_list: + in truth_copy_output_list: job.write(f'if [ -f "{truth_output_file_tuple[0]}" ]; then ' +f"cp -v {truth_output_file_tuple[0]} " +f"{truth_output_file_tuple[1]}" @@ -231,6 +243,7 @@ for model_idx in range(len(model_list)): job_env_dict['MODEL'] = model_list[model_idx] njobs = (int(njobs) + 1) + job_env_dict['job_num'] = str(njobs) # Create job file job_file = os.path.join(JOB_GROUP_jobs_dir, 'job'+str(njobs)) print("Creating job script: "+job_file) @@ -238,6 +251,15 @@ job.write('#!/bin/bash\n') job.write('set -x\n') job.write('\n') + # Create job working directory + job_env_dict['job_num_work_dir'] = os.path.join( + DATA, f"{VERIF_CASE}_{STEP}", 'METplus_output', + 'job_work_dir', JOB_GROUP, + f"job{job_env_dict['job_num']}" + ) + job_env_dict['MET_TMP_DIR'] = os.path.join( + job_env_dict['job_num_work_dir'], 'tmp' + ) # Set any environment variables for special cases if JOB_GROUP == 'reformat_data': if verif_type == 'prepbufr': @@ -266,7 +288,7 @@ check_truth_files = False if check_model_files: (model_files_exist, valid_date_fhr_list, - model_copy_output_DATA2COMOUT_list) = ( + model_copy_output_list) = ( sub_util.check_weeks3_4_model_files(job_env_dict) ) job_env_dict['fhr_list'] = ( @@ -277,7 +299,7 @@ job_env_dict.pop('fhr_inc') if check_truth_files: (all_truth_file_exist, - truth_copy_output_DATA2COMOUT_list) = ( + truth_copy_output_list) = ( sub_util.check_weeks3_4_truth_files(job_env_dict) ) if model_files_exist and all_truth_file_exist: @@ -295,14 +317,16 @@ job.write('\n') # Write job commands if write_job_cmds: + if not os.path.exists(job_env_dict['job_num_work_dir']): + os.makedirs(job_env_dict['job_num_work_dir']) for cmd in verif_type_job_commands_list: job.write(cmd+'\n') job.write('export err=$?; err_chk'+'\n') - # Copy DATA files to COMOUT restart dir + # Copy files to COMOUT restart dir # to be used in possible restart if job_env_dict['SENDCOM'] == 'YES': for model_output_file_tuple \ - in model_copy_output_DATA2COMOUT_list: + in model_copy_output_list: job.write(f'if [ -f "{model_output_file_tuple[0]}" ]; then ' +f"cp -v {model_output_file_tuple[0]} " +f"{model_output_file_tuple[1]}" diff --git a/ush/subseasonal/subseasonal_util.py b/ush/subseasonal/subseasonal_util.py index 268642a5a..7b8d3b4da 100644 --- a/ush/subseasonal/subseasonal_util.py +++ b/ush/subseasonal/subseasonal_util.py @@ -1311,8 +1311,7 @@ def check_daily_model_files(job_dict): exist or not (boolean) fhr_list - list of forecast hours that model files exist for (string) - model_copy_output_DATA2COMOUT_list - list of files to copy from - DATA to COMOUT + model_copy_output_list - list of files to copy """ valid_date_dt = datetime.datetime.strptime( job_dict['DATE']+job_dict['valid_hr_start'], @@ -1344,16 +1343,8 @@ def check_daily_model_files(job_dict): model+'.ens'+mb +'.{init?fmt=%Y%m%d%H}.' +'f{lead?fmt=%3H}') - output_DATA_file_format = os.path.join( - verif_case_dir, 'METplus_output', - job_dict['RUN']+'.'+job_dict['DATE'], - job_dict['MODEL'], job_dict['VERIF_CASE'], - 'grid_stat_'+job_dict['VERIF_TYPE']+'_' - +job_dict['job_name']+'_{lead?fmt=%2H}' - '0000L_{valid?fmt=%Y%m%d_%H%M%S}V_pairs.nc' - ) - output_COMOUT_file_format = os.path.join( - job_dict['COMOUT'], + output_JOB_file_format = os.path.join( + job_dict['job_num_work_dir'], job_dict['RUN']+'.'+job_dict['DATE'], job_dict['MODEL'], job_dict['VERIF_CASE'], 'grid_stat_'+job_dict['VERIF_TYPE']+'_' @@ -1399,22 +1390,30 @@ def check_daily_model_files(job_dict): ) input_fhr_list = copy.deepcopy(fhr_list) # Check output files - model_copy_output_DATA2COMOUT_list = [] + output_DATA_file_format = output_JOB_file_format.replace( + job_dict['job_num_work_dir'], + os.path.join(verif_case_dir, 'METplus_output') + ) + output_COMOUT_file_format = output_JOB_file_format.replace( + job_dict['job_num_work_dir'], job_dict['COMOUT'] + ) + model_copy_output_list = [] for fhr_key in list(fhr_check_output_dict.keys()): for fhr_fileN_key in list(fhr_check_output_dict[fhr_key].keys()): - fhr_fileN_DATA = format_filler( - output_DATA_file_format, + fhr_fileN_JOB = format_filler( + output_JOB_file_format, fhr_check_output_dict[fhr_key][fhr_fileN_key]['valid_date'], fhr_check_output_dict[fhr_key][fhr_fileN_key]['init_date'], fhr_check_output_dict[fhr_key][fhr_fileN_key]['forecast_hour'], {} ) - fhr_fileN_COMOUT = format_filler( - output_COMOUT_file_format, - fhr_check_output_dict[fhr_key][fhr_fileN_key]['valid_date'], - fhr_check_output_dict[fhr_key][fhr_fileN_key]['init_date'], - fhr_check_output_dict[fhr_key][fhr_fileN_key]['forecast_hour'], - {} + fhr_fileN_DATA = fhr_fileN_JOB.replace( + job_dict['job_num_work_dir'], + os.path.join(verif_case_dir, 'METplus_output') + ) + fhr_fileN_COMOUT = fhr_fileN_JOB.replace( + job_dict['job_num_work_dir'], + job_dict['COMOUT'] ) if os.path.exists(fhr_fileN_COMOUT): # Copy restart files from COMOUT to DATA dir @@ -1433,16 +1432,16 @@ def check_daily_model_files(job_dict): if fhr_check_output_dict[fhr_key]\ [fhr_fileN_key]['forecast_hour'] \ in fhr_list: - if (fhr_fileN_DATA, fhr_fileN_COMOUT) \ - not in model_copy_output_DATA2COMOUT_list: - model_copy_output_DATA2COMOUT_list.append( - (fhr_fileN_DATA, fhr_fileN_COMOUT) + if (fhr_fileN_JOB, fhr_fileN_COMOUT) \ + not in model_copy_output_list: + model_copy_output_list.append( + (fhr_fileN_JOB, fhr_fileN_COMOUT) ) if len(input_fhr_list) == 3: model_files_exist = True else: model_files_exist = False - return model_files_exist, fhr_list, model_copy_output_DATA2COMOUT_list + return model_files_exist, fhr_list, model_copy_output_list def check_weekly_model_files(job_dict): @@ -1457,8 +1456,7 @@ def check_weekly_model_files(job_dict): exist or not (boolean) fhr_list - list of forecast hours that model files exist for (string) - model_copy_output_DATA2COMOUT_list - list of files to copy from - DATA to COMOUT + model_copy_output_list - list of files to copy """ valid_date_dt = datetime.datetime.strptime( job_dict['DATE']+job_dict['valid_hr_start'], @@ -1507,16 +1505,8 @@ def check_weekly_model_files(job_dict): +'.{init?fmt=%Y%m%d%H}.' +'f{lead?fmt=%3H}') if job_dict['VERIF_CASE'] == 'grid2grid': - output_DATA_file_format = os.path.join( - verif_case_dir, 'METplus_output', - job_dict['RUN']+'.'+job_dict['DATE'], - job_dict['MODEL'], job_dict['VERIF_CASE'], - 'grid_stat_'+job_dict['VERIF_TYPE']+'_' - +job_dict['job_name']+'_{lead?fmt=%2H}' - '0000L_{valid?fmt=%Y%m%d_%H%M%S}V_pairs.nc' - ) - output_COMOUT_file_format = os.path.join( - job_dict['COMOUT'], + output_JOB_file_format = os.path.join( + job_dict['job_num_work_dir'], job_dict['RUN']+'.'+job_dict['DATE'], job_dict['MODEL'], job_dict['VERIF_CASE'], 'grid_stat_'+job_dict['VERIF_TYPE']+'_' @@ -1524,17 +1514,8 @@ def check_weekly_model_files(job_dict): '0000L_{valid?fmt=%Y%m%d_%H%M%S}V_pairs.nc' ) if job_dict['VERIF_CASE'] == 'grid2obs': - output_DATA_file_format = os.path.join( - verif_case_dir, 'METplus_output', - job_dict['RUN']+'.'+job_dict['DATE'], - job_dict['MODEL'], job_dict['VERIF_CASE'], - 'gen_ens_prod_'+job_dict['VERIF_TYPE']+'_' - +'GenEnsProd_FHR{lead?fmt=%3H}' - +'_{valid?fmt=%Y%m%d}' - +'_{valid?fmt=%H}0000V_ens.nc' - ) - output_COMOUT_file_format = os.path.join( - job_dict['COMOUT'], + output_JOB_file_format = os.path.join( + job_dict['job_num_work_dir'], job_dict['RUN']+'.'+job_dict['DATE'], job_dict['MODEL'], job_dict['VERIF_CASE'], 'gen_ens_prod_'+job_dict['VERIF_TYPE']+'_' @@ -1580,16 +1561,8 @@ def check_weekly_model_files(job_dict): +'_{valid?fmt=%Y%m%d}' +'_{valid?fmt=%H}0000V' +'_ens.nc') - output_DATA_file_format = os.path.join( - verif_case_dir, 'METplus_output', - job_dict['RUN']+'.'+job_dict['DATE'], - job_dict['MODEL'], job_dict['VERIF_CASE'], - 'point_stat_'+job_dict['VERIF_TYPE']+'_' - +job_dict['job_name']+'_{lead?fmt=%2H}' - '0000L_{valid?fmt=%Y%m%d_%H%M%S}V.stat' - ) - output_COMOUT_file_format = os.path.join( - job_dict['COMOUT'], + output_JOB_file_format = os.path.join( + job_dict['job_num_work_dir'], job_dict['RUN']+'.'+job_dict['DATE'], job_dict['MODEL'], job_dict['VERIF_CASE'], 'point_stat_'+job_dict['VERIF_TYPE']+'_' @@ -1666,22 +1639,30 @@ def check_weekly_model_files(job_dict): ) input_fhr_list = copy.deepcopy(fhr_list) # Check output files - model_copy_output_DATA2COMOUT_list = [] + output_DATA_file_format = output_JOB_file_format.replace( + job_dict['job_num_work_dir'], + os.path.join(verif_case_dir, 'METplus_output') + ) + output_COMOUT_file_format = output_JOB_file_format.replace( + job_dict['job_num_work_dir'], job_dict['COMOUT'] + ) + model_copy_output_list = [] for fhr_key in list(fhr_check_output_dict.keys()): for fhr_fileN_key in list(fhr_check_output_dict[fhr_key].keys()): - fhr_fileN_DATA = format_filler( - output_DATA_file_format, + fhr_fileN_JOB = format_filler( + output_JOB_file_format, fhr_check_output_dict[fhr_key][fhr_fileN_key]['valid_date'], fhr_check_output_dict[fhr_key][fhr_fileN_key]['init_date'], fhr_check_output_dict[fhr_key][fhr_fileN_key]['forecast_hour'], {} ) - fhr_fileN_COMOUT = format_filler( - output_COMOUT_file_format, - fhr_check_output_dict[fhr_key][fhr_fileN_key]['valid_date'], - fhr_check_output_dict[fhr_key][fhr_fileN_key]['init_date'], - fhr_check_output_dict[fhr_key][fhr_fileN_key]['forecast_hour'], - {} + fhr_fileN_DATA = fhr_fileN_JOB.replace( + job_dict['job_num_work_dir'], + os.path.join(verif_case_dir, 'METplus_output') + ) + fhr_fileN_COMOUT = fhr_fileN_JOB.replace( + job_dict['job_num_work_dir'], + job_dict['COMOUT'] ) if os.path.exists(fhr_fileN_COMOUT): # Copy restart files from COMOUT to DATA dir @@ -1700,10 +1681,10 @@ def check_weekly_model_files(job_dict): if fhr_check_output_dict[fhr_key]\ [fhr_fileN_key]['forecast_hour'] \ in fhr_list: - if (fhr_fileN_DATA, fhr_fileN_COMOUT) \ - not in model_copy_output_DATA2COMOUT_list: - model_copy_output_DATA2COMOUT_list.append( - (fhr_fileN_DATA, fhr_fileN_COMOUT) + if (fhr_fileN_JOB, fhr_fileN_COMOUT) \ + not in model_copy_output_list: + model_copy_output_list.append( + (fhr_fileN_JOB, fhr_fileN_COMOUT) ) if len(input_fhr_list) >= 12: model_files_exist = True @@ -1711,7 +1692,7 @@ def check_weekly_model_files(job_dict): model_files_exist = False if len(fhr_list) == 0: model_files_exist = False - return model_files_exist, fhr_list, model_copy_output_DATA2COMOUT_list + return model_files_exist, fhr_list, model_copy_output_list def check_monthly_model_files(job_dict): @@ -1726,8 +1707,7 @@ def check_monthly_model_files(job_dict): exist or not (boolean) fhr_list - list of forecast hours that model files exist for (string) - model_copy_output_DATA2COMOUT_list - list of files to copy from - DATA to COMOUT + model_copy_output_list - list of files to copy """ valid_date_dt = datetime.datetime.strptime( job_dict['DATE']+job_dict['valid_hr_start'], @@ -1760,16 +1740,8 @@ def check_monthly_model_files(job_dict): model+'.ens'+mb +'.{init?fmt=%Y%m%d%H}.' +'f{lead?fmt=%3H}') - output_DATA_file_format = os.path.join( - verif_case_dir, 'METplus_output', - job_dict['RUN']+'.'+job_dict['DATE'], - job_dict['MODEL'], job_dict['VERIF_CASE'], - 'grid_stat_'+job_dict['VERIF_TYPE']+'_' - +job_dict['job_name']+'_{lead?fmt=%2H}' - '0000L_{valid?fmt=%Y%m%d_%H%M%S}V_pairs.nc' - ) - output_COMOUT_file_format = os.path.join( - job_dict['COMOUT'], + output_JOB_file_format = os.path.join( + job_dict['job_num_work_dir'], job_dict['RUN']+'.'+job_dict['DATE'], job_dict['MODEL'], job_dict['VERIF_CASE'], 'grid_stat_'+job_dict['VERIF_TYPE']+'_' @@ -1816,22 +1788,30 @@ def check_monthly_model_files(job_dict): ) input_fhr_list = copy.deepcopy(fhr_list) # Check output files - model_copy_output_DATA2COMOUT_list = [] + output_DATA_file_format = output_JOB_file_format.replace( + job_dict['job_num_work_dir'], + os.path.join(verif_case_dir, 'METplus_output') + ) + output_COMOUT_file_format = output_JOB_file_format.replace( + job_dict['job_num_work_dir'], job_dict['COMOUT'] + ) + model_copy_output_list = [] for fhr_key in list(fhr_check_output_dict.keys()): for fhr_fileN_key in list(fhr_check_output_dict[fhr_key].keys()): - fhr_fileN_DATA = format_filler( - output_DATA_file_format, + fhr_fileN_JOB = format_filler( + output_JOB_file_format, fhr_check_output_dict[fhr_key][fhr_fileN_key]['valid_date'], fhr_check_output_dict[fhr_key][fhr_fileN_key]['init_date'], fhr_check_output_dict[fhr_key][fhr_fileN_key]['forecast_hour'], {} ) - fhr_fileN_COMOUT = format_filler( - output_COMOUT_file_format, - fhr_check_output_dict[fhr_key][fhr_fileN_key]['valid_date'], - fhr_check_output_dict[fhr_key][fhr_fileN_key]['init_date'], - fhr_check_output_dict[fhr_key][fhr_fileN_key]['forecast_hour'], - {} + fhr_fileN_DATA = fhr_fileN_JOB.replace( + job_dict['job_num_work_dir'], + os.path.join(verif_case_dir, 'METplus_output') + ) + fhr_fileN_COMOUT = fhr_fileN_JOB.replace( + job_dict['job_num_work_dir'], + job_dict['COMOUT'] ) if os.path.exists(fhr_fileN_COMOUT): # Copy restart files from COMOUT to DATA dir @@ -1850,16 +1830,16 @@ def check_monthly_model_files(job_dict): if fhr_check_output_dict[fhr_key]\ [fhr_fileN_key]['forecast_hour'] \ in fhr_list: - if (fhr_fileN_DATA, fhr_fileN_COMOUT) \ - not in model_copy_output_DATA2COMOUT_list: - model_copy_output_DATA2COMOUT_list.append( - (fhr_fileN_DATA, fhr_fileN_COMOUT) + if (fhr_fileN_JOB, fhr_fileN_COMOUT) \ + not in model_copy_output_list: + model_copy_output_list.append( + (fhr_fileN_JOB, fhr_fileN_COMOUT) ) if len(input_fhr_list) >= 49: model_files_exist = True else: model_files_exist = False - return model_files_exist, fhr_list, model_copy_output_DATA2COMOUT_list + return model_files_exist, fhr_list, model_copy_output_list def check_days6_10_model_files(job_dict): @@ -1874,8 +1854,7 @@ def check_days6_10_model_files(job_dict): exist or not (boolean) fhr_list - list of forecast hours that model files exist for (string) - model_copy_output_DATA2COMOUT_list - list of files to copy from - DATA to COMOUT + model_copy_output_list - list of files to copy """ valid_date_dt = datetime.datetime.strptime( job_dict['DATE']+job_dict['valid_hr_start'], @@ -1921,16 +1900,8 @@ def check_days6_10_model_files(job_dict): +'.{init?fmt=%Y%m%d%H}.' +'f{lead?fmt=%3H}') if job_dict['VERIF_CASE'] == 'grid2grid': - output_DATA_file_format = os.path.join( - verif_case_dir, 'METplus_output', - job_dict['RUN']+'.'+job_dict['DATE'], - job_dict['MODEL'], job_dict['VERIF_CASE'], - 'grid_stat_'+job_dict['VERIF_TYPE']+'_' - +job_dict['job_name']+'_{lead?fmt=%2H}' - '0000L_{valid?fmt=%Y%m%d_%H%M%S}V_pairs.nc' - ) - output_COMOUT_file_format = os.path.join( - job_dict['COMOUT'], + output_JOB_file_format = os.path.join( + job_dict['job_num_work_dir'], job_dict['RUN']+'.'+job_dict['DATE'], job_dict['MODEL'], job_dict['VERIF_CASE'], 'grid_stat_'+job_dict['VERIF_TYPE']+'_' @@ -1938,17 +1909,8 @@ def check_days6_10_model_files(job_dict): '0000L_{valid?fmt=%Y%m%d_%H%M%S}V_pairs.nc' ) if job_dict['VERIF_CASE'] == 'grid2obs': - output_DATA_file_format = os.path.join( - verif_case_dir, 'METplus_output', - job_dict['RUN']+'.'+job_dict['DATE'], - job_dict['MODEL'], job_dict['VERIF_CASE'], - 'gen_ens_prod_'+job_dict['VERIF_TYPE']+'_' - +'GenEnsProd_FHR{lead?fmt=%3H}' - +'_{valid?fmt=%Y%m%d}' - +'_{valid?fmt=%H}0000V_ens.nc' - ) - output_COMOUT_file_format = os.path.join( - job_dict['COMOUT'], + output_JOB_file_format = os.path.join( + job_dict['job_num_work_dir'], job_dict['RUN']+'.'+job_dict['DATE'], job_dict['MODEL'], job_dict['VERIF_CASE'], 'gen_ens_prod_'+job_dict['VERIF_TYPE']+'_' @@ -1994,16 +1956,8 @@ def check_days6_10_model_files(job_dict): +'_{valid?fmt=%Y%m%d}' +'_{valid?fmt=%H}0000V' +'_ens.nc') - output_DATA_file_format = os.path.join( - verif_case_dir, 'METplus_output', - job_dict['RUN']+'.'+job_dict['DATE'], - job_dict['MODEL'], job_dict['VERIF_CASE'], - 'point_stat_'+job_dict['VERIF_TYPE']+'_' - +job_dict['job_name']+'_{lead?fmt=%2H}' - '0000L_{valid?fmt=%Y%m%d_%H%M%S}V.stat' - ) - output_COMOUT_file_format = os.path.join( - job_dict['COMOUT'], + output_JOB_file_format = os.path.join( + job_dict['job_num_work_dir'], job_dict['RUN']+'.'+job_dict['DATE'], job_dict['MODEL'], job_dict['VERIF_CASE'], 'point_stat_'+job_dict['VERIF_TYPE']+'_' @@ -2078,22 +2032,30 @@ def check_days6_10_model_files(job_dict): ) input_fhr_list = copy.deepcopy(fhr_list) # Check output files - model_copy_output_DATA2COMOUT_list = [] + output_DATA_file_format = output_JOB_file_format.replace( + job_dict['job_num_work_dir'], + os.path.join(verif_case_dir, 'METplus_output') + ) + output_COMOUT_file_format = output_JOB_file_format.replace( + job_dict['job_num_work_dir'], job_dict['COMOUT'] + ) + model_copy_output_list = [] for fhr_key in list(fhr_check_output_dict.keys()): for fhr_fileN_key in list(fhr_check_output_dict[fhr_key].keys()): - fhr_fileN_DATA = format_filler( - output_DATA_file_format, + fhr_fileN_JOB = format_filler( + output_JOB_file_format, fhr_check_output_dict[fhr_key][fhr_fileN_key]['valid_date'], fhr_check_output_dict[fhr_key][fhr_fileN_key]['init_date'], fhr_check_output_dict[fhr_key][fhr_fileN_key]['forecast_hour'], {} ) - fhr_fileN_COMOUT = format_filler( - output_COMOUT_file_format, - fhr_check_output_dict[fhr_key][fhr_fileN_key]['valid_date'], - fhr_check_output_dict[fhr_key][fhr_fileN_key]['init_date'], - fhr_check_output_dict[fhr_key][fhr_fileN_key]['forecast_hour'], - {} + fhr_fileN_DATA = fhr_fileN_JOB.replace( + job_dict['job_num_work_dir'], + os.path.join(verif_case_dir, 'METplus_output') + ) + fhr_fileN_COMOUT = fhr_fileN_JOB.replace( + job_dict['job_num_work_dir'], + job_dict['COMOUT'] ) if os.path.exists(fhr_fileN_COMOUT): # Copy restart files from COMOUT to DATA dir @@ -2112,10 +2074,10 @@ def check_days6_10_model_files(job_dict): if fhr_check_output_dict[fhr_key]\ [fhr_fileN_key]['forecast_hour'] \ in fhr_list: - if (fhr_fileN_DATA, fhr_fileN_COMOUT) \ - not in model_copy_output_DATA2COMOUT_list: - model_copy_output_DATA2COMOUT_list.append( - (fhr_fileN_DATA, fhr_fileN_COMOUT) + if (fhr_fileN_JOB, fhr_fileN_COMOUT) \ + not in model_copy_output_list: + model_copy_output_list.append( + (fhr_fileN_JOB, fhr_fileN_COMOUT) ) if len(input_fhr_list) >= 9: model_files_exist = True @@ -2123,7 +2085,7 @@ def check_days6_10_model_files(job_dict): model_files_exist = False if len(fhr_list) == 0: model_files_exist = False - return model_files_exist, fhr_list, model_copy_output_DATA2COMOUT_list + return model_files_exist, fhr_list, model_copy_output_list def check_weeks3_4_model_files(job_dict): @@ -2138,8 +2100,7 @@ def check_weeks3_4_model_files(job_dict): exist or not (boolean) fhr_list - list of forecast hours that model files exist for (string) - model_copy_output_DATA2COMOUT_list - list of files to copy from - DATA to COMOUT + model_copy_output_list - list of files to copy """ valid_date_dt = datetime.datetime.strptime( job_dict['DATE']+job_dict['valid_hr_start'], @@ -2185,16 +2146,8 @@ def check_weeks3_4_model_files(job_dict): +'.{init?fmt=%Y%m%d%H}.' +'f{lead?fmt=%3H}') if job_dict['VERIF_CASE'] == 'grid2grid': - output_DATA_file_format = os.path.join( - verif_case_dir, 'METplus_output', - job_dict['RUN']+'.'+job_dict['DATE'], - job_dict['MODEL'], job_dict['VERIF_CASE'], - 'grid_stat_'+job_dict['VERIF_TYPE']+'_' - +job_dict['job_name']+'_{lead?fmt=%2H}' - '0000L_{valid?fmt=%Y%m%d_%H%M%S}V_pairs.nc' - ) - output_COMOUT_file_format = os.path.join( - job_dict['COMOUT'], + output_JOB_file_format = os.path.join( + job_dict['job_num_work_dir'], job_dict['RUN']+'.'+job_dict['DATE'], job_dict['MODEL'], job_dict['VERIF_CASE'], 'grid_stat_'+job_dict['VERIF_TYPE']+'_' @@ -2202,17 +2155,8 @@ def check_weeks3_4_model_files(job_dict): '0000L_{valid?fmt=%Y%m%d_%H%M%S}V_pairs.nc' ) if job_dict['VERIF_CASE'] == 'grid2obs': - output_DATA_file_format = os.path.join( - verif_case_dir, 'METplus_output', - job_dict['RUN']+'.'+job_dict['DATE'], - job_dict['MODEL'], job_dict['VERIF_CASE'], - 'gen_ens_prod_'+job_dict['VERIF_TYPE']+'_' - +'GenEnsProd_FHR{lead?fmt=%3H}' - +'_{valid?fmt=%Y%m%d}' - +'_{valid?fmt=%H}0000V_ens.nc' - ) - output_COMOUT_file_format = os.path.join( - job_dict['COMOUT'], + output_JOB_file_format = os.path.join( + job_dict['job_num_work_dir'], job_dict['RUN']+'.'+job_dict['DATE'], job_dict['MODEL'], job_dict['VERIF_CASE'], 'gen_ens_prod_'+job_dict['VERIF_TYPE']+'_' @@ -2258,16 +2202,8 @@ def check_weeks3_4_model_files(job_dict): +'_{valid?fmt=%Y%m%d}' +'_{valid?fmt=%H}0000V' +'_ens.nc') - output_DATA_file_format = os.path.join( - verif_case_dir, 'METplus_output', - job_dict['RUN']+'.'+job_dict['DATE'], - job_dict['MODEL'], job_dict['VERIF_CASE'], - 'point_stat_'+job_dict['VERIF_TYPE']+'_' - +job_dict['job_name']+'_{lead?fmt=%2H}' - '0000L_{valid?fmt=%Y%m%d_%H%M%S}V.stat' - ) - output_COMOUT_file_format = os.path.join( - job_dict['COMOUT'], + output_JOB_file_format = os.path.join( + job_dict['job_num_work_dir'], job_dict['RUN']+'.'+job_dict['DATE'], job_dict['MODEL'], job_dict['VERIF_CASE'], 'point_stat_'+job_dict['VERIF_TYPE']+'_' @@ -2342,22 +2278,30 @@ def check_weeks3_4_model_files(job_dict): ) input_fhr_list = copy.deepcopy(fhr_list) # Check output files - model_copy_output_DATA2COMOUT_list = [] + output_DATA_file_format = output_JOB_file_format.replace( + job_dict['job_num_work_dir'], + os.path.join(verif_case_dir, 'METplus_output') + ) + output_COMOUT_file_format = output_JOB_file_format.replace( + job_dict['job_num_work_dir'], job_dict['COMOUT'] + ) + model_copy_output_list = [] for fhr_key in list(fhr_check_output_dict.keys()): for fhr_fileN_key in list(fhr_check_output_dict[fhr_key].keys()): - fhr_fileN_DATA = format_filler( - output_DATA_file_format, + fhr_fileN_JOB = format_filler( + output_JOB_file_format, fhr_check_output_dict[fhr_key][fhr_fileN_key]['valid_date'], fhr_check_output_dict[fhr_key][fhr_fileN_key]['init_date'], fhr_check_output_dict[fhr_key][fhr_fileN_key]['forecast_hour'], {} ) - fhr_fileN_COMOUT = format_filler( - output_COMOUT_file_format, - fhr_check_output_dict[fhr_key][fhr_fileN_key]['valid_date'], - fhr_check_output_dict[fhr_key][fhr_fileN_key]['init_date'], - fhr_check_output_dict[fhr_key][fhr_fileN_key]['forecast_hour'], - {} + fhr_fileN_DATA = fhr_fileN_JOB.replace( + job_dict['job_num_work_dir'], + os.path.join(verif_case_dir, 'METplus_output') + ) + fhr_fileN_COMOUT = fhr_fileN_JOB.replace( + job_dict['job_num_work_dir'], + job_dict['COMOUT'] ) if os.path.exists(fhr_fileN_COMOUT): # Copy restart files from COMOUT to DATA dir @@ -2376,10 +2320,10 @@ def check_weeks3_4_model_files(job_dict): if fhr_check_output_dict[fhr_key]\ [fhr_fileN_key]['forecast_hour'] \ in fhr_list: - if (fhr_fileN_DATA, fhr_fileN_COMOUT) \ - not in model_copy_output_DATA2COMOUT_list: - model_copy_output_DATA2COMOUT_list.append( - (fhr_fileN_DATA, fhr_fileN_COMOUT) + if (fhr_fileN_JOB, fhr_fileN_COMOUT) \ + not in model_copy_output_list: + model_copy_output_list.append( + (fhr_fileN_JOB, fhr_fileN_COMOUT) ) if len(input_fhr_list) >= 23: model_files_exist = True @@ -2387,7 +2331,7 @@ def check_weeks3_4_model_files(job_dict): model_files_exist = False if len(fhr_list) == 0: model_files_exist = False - return model_files_exist, fhr_list, model_copy_output_DATA2COMOUT_list + return model_files_exist, fhr_list, model_copy_output_list def check_model_files(job_dict): @@ -2402,8 +2346,7 @@ def check_model_files(job_dict): exist or not (boolean) fhr_list - list of forecast hours that model files exist for (string) - model_copy_output_DATA2COMOUT_list - list of files to copy from - DATA to COMOUT + model_copy_output_list - list of files to copy """ valid_date_dt = datetime.datetime.strptime( job_dict['DATE']+job_dict['valid_hr_start'], @@ -2470,17 +2413,8 @@ def check_model_files(job_dict): +'_init' +'{init?fmt=%Y%m%d%H}_' +'fhr{lead?fmt=%3H}.nc') - output_DATA_file_format = os.path.join( - verif_case_dir, 'METplus_output', - job_dict['RUN']+'.'+job_dict['DATE'], - model, job_dict['VERIF_CASE'], 'weekly_avg_' - +job_dict['VERIF_TYPE']+'_'+job_dict['job_name'] - +'_init{init?fmt=%Y%m%d%H}_' - +'valid{valid_shift?fmt=%Y%m%d%H?shift=-168}' - +'to{valid?fmt=%Y%m%d%H}.nc' - ) - output_COMOUT_file_format = os.path.join( - job_dict['COMOUT'], + output_JOB_file_format = os.path.join( + job_dict['job_num_work_dir'], job_dict['RUN']+'.'+job_dict['DATE'], model, job_dict['VERIF_CASE'], 'weekly_avg_' +job_dict['VERIF_TYPE']+'_'+job_dict['job_name'] @@ -2505,17 +2439,8 @@ def check_model_files(job_dict): +'_init' +'{init?fmt=%Y%m%d%H}_' +'fhr{lead?fmt=%3H}.nc') - output_DATA_file_format = os.path.join( - verif_case_dir, 'METplus_output', - job_dict['RUN']+'.'+job_dict['DATE'], - model, job_dict['VERIF_CASE'], 'days6_10_avg_' - +job_dict['VERIF_TYPE']+'_'+job_dict['job_name'] - +'_init{init?fmt=%Y%m%d%H}_' - +'valid{valid_shift?fmt=%Y%m%d%H?shift=-120}' - +'to{valid?fmt=%Y%m%d%H}.nc' - ) - output_COMOUT_file_format = os.path.join( - job_dict['COMOUT'], + output_JOB_file_format = os.path.join( + job_dict['job_num_work_dir'], job_dict['RUN']+'.'+job_dict['DATE'], model, job_dict['VERIF_CASE'], 'days6_10_avg_' +job_dict['VERIF_TYPE']+'_'+job_dict['job_name'] @@ -2540,17 +2465,8 @@ def check_model_files(job_dict): +'_init' +'{init?fmt=%Y%m%d%H}_' +'fhr{lead?fmt=%3H}.nc') - output_DATA_file_format = os.path.join( - verif_case_dir, 'METplus_output', - job_dict['RUN']+'.'+job_dict['DATE'], - model, job_dict['VERIF_CASE'], 'weeks3_4_avg_' - +job_dict['VERIF_TYPE']+'_'+job_dict['job_name'] - +'_init{init?fmt=%Y%m%d%H}_' - +'valid{valid_shift?fmt=%Y%m%d%H?shift=-336}' - +'to{valid?fmt=%Y%m%d%H}.nc' - ) - output_COMOUT_file_format = os.path.join( - job_dict['COMOUT'], + output_JOB_file_format = os.path.join( + job_dict['job_num_work_dir'], job_dict['RUN']+'.'+job_dict['DATE'], model, job_dict['VERIF_CASE'], 'weeks3_4_avg_' +job_dict['VERIF_TYPE']+'_'+job_dict['job_name'] @@ -2575,17 +2491,8 @@ def check_model_files(job_dict): +'{valid?fmt=%Y%m%d}_' +'{valid?fmt=%H}0000V_' +'pairs.nc') - output_DATA_file_format = os.path.join( - verif_case_dir, 'METplus_output', - job_dict['RUN']+'.'+job_dict['DATE'], - model, job_dict['VERIF_CASE'], 'daily_avg_' - +job_dict['VERIF_TYPE']+'_'+job_dict['job_name'] - +'_init{init?fmt=%Y%m%d%H}_' - +'valid{valid_shift?fmt=%Y%m%d%H?shift=-24}' - +'to{valid?fmt=%Y%m%d%H}.nc' - ) - output_COMOUT_file_format = os.path.join( - job_dict['COMOUT'], + output_JOB_file_format = os.path.join( + job_dict['job_num_work_dir'], job_dict['RUN']+'.'+job_dict['DATE'], model, job_dict['VERIF_CASE'], 'daily_avg_' +job_dict['VERIF_TYPE']+'_'+job_dict['job_name'] @@ -2611,17 +2518,8 @@ def check_model_files(job_dict): +'{valid?fmt=%Y%m%d}_' +'{valid?fmt=%H}0000V_' +'pairs.nc') - output_DATA_file_format = os.path.join( - verif_case_dir, 'METplus_output', - job_dict['RUN']+'.'+job_dict['DATE'], - model, job_dict['VERIF_CASE'], 'weekly_avg_' - +job_dict['VERIF_TYPE']+'_'+job_dict['job_name'] - +'_init{init?fmt=%Y%m%d%H}_' - +'valid{valid_shift?fmt=%Y%m%d%H?shift=-168}' - +'to{valid?fmt=%Y%m%d%H}.nc' - ) - output_COMOUT_file_format = os.path.join( - job_dict['COMOUT'], + output_JOB_file_format = os.path.join( + job_dict['job_num_work_dir'], job_dict['RUN']+'.'+job_dict['DATE'], model, job_dict['VERIF_CASE'], 'weekly_avg_' +job_dict['VERIF_TYPE']+'_'+job_dict['job_name'] @@ -2647,17 +2545,8 @@ def check_model_files(job_dict): +'{valid?fmt=%Y%m%d}_' +'{valid?fmt=%H}0000V_' +'pairs.nc') - output_DATA_file_format = os.path.join( - verif_case_dir, 'METplus_output', - job_dict['RUN']+'.'+job_dict['DATE'], - model, job_dict['VERIF_CASE'], 'monthly_avg_' - +job_dict['VERIF_TYPE']+'_'+job_dict['job_name'] - +'_init{init?fmt=%Y%m%d%H}_' - +'valid{valid_shift?fmt=%Y%m%d%H?shift=-720}' - +'to{valid?fmt=%Y%m%d%H}.nc' - ) - output_COMOUT_file_format = os.path.join( - job_dict['COMOUT'], + output_JOB_file_format = os.path.join( + job_dict['job_num_work_dir'], job_dict['RUN']+'.'+job_dict['DATE'], model, job_dict['VERIF_CASE'], 'monthly_avg_' +job_dict['VERIF_TYPE']+'_'+job_dict['job_name'] @@ -2680,17 +2569,8 @@ def check_model_files(job_dict): +'{valid?fmt=%Y%m%d}_' +'{valid?fmt=%H}0000V_' +'pairs.nc') - output_DATA_file_format = os.path.join( - verif_case_dir, 'METplus_output', - job_dict['RUN']+'.'+job_dict['DATE'], - model, job_dict['VERIF_CASE'], 'weekly_avg_' - +job_dict['VERIF_TYPE']+'_'+job_dict['job_name'] - +'_init{init?fmt=%Y%m%d%H}_' - +'valid{valid_shift?fmt=%Y%m%d%H?shift=-168}' - +'to{valid?fmt=%Y%m%d%H}.nc' - ) - output_COMOUT_file_format = os.path.join( - job_dict['COMOUT'], + output_JOB_file_format = os.path.join( + job_dict['job_num_work_dir'], job_dict['RUN']+'.'+job_dict['DATE'], model, job_dict['VERIF_CASE'], 'weekly_avg_' +job_dict['VERIF_TYPE']+'_'+job_dict['job_name'] @@ -2713,17 +2593,8 @@ def check_model_files(job_dict): +'{valid?fmt=%Y%m%d}_' +'{valid?fmt=%H}0000V_' +'pairs.nc') - output_DATA_file_format = os.path.join( - verif_case_dir, 'METplus_output', - job_dict['RUN']+'.'+job_dict['DATE'], - model, job_dict['VERIF_CASE'], 'weekly_avg_' - +job_dict['VERIF_TYPE']+'_'+job_dict['job_name'] - +'_init{init?fmt=%Y%m%d%H}_' - +'valid{valid_shift?fmt=%Y%m%d%H?shift=-168}' - +'to{valid?fmt=%Y%m%d%H}.nc' - ) - output_COMOUT_file_format = os.path.join( - job_dict['COMOUT'], + output_JOB_file_format = os.path.join( + job_dict['job_num_work_dir'], job_dict['RUN']+'.'+job_dict['DATE'], model, job_dict['VERIF_CASE'], 'weekly_avg_' +job_dict['VERIF_TYPE']+'_'+job_dict['job_name'] @@ -2746,17 +2617,8 @@ def check_model_files(job_dict): +'{valid?fmt=%Y%m%d}_' +'{valid?fmt=%H}0000V_' +'pairs.nc') - output_DATA_file_format = os.path.join( - verif_case_dir, 'METplus_output', - job_dict['RUN']+'.'+job_dict['DATE'], - model, job_dict['VERIF_CASE'], 'days6_10_avg_' - +job_dict['VERIF_TYPE']+'_'+job_dict['job_name'] - +'_init{init?fmt=%Y%m%d%H}_' - +'valid{valid_shift?fmt=%Y%m%d%H?shift=-120}' - +'to{valid?fmt=%Y%m%d%H}.nc' - ) - output_COMOUT_file_format = os.path.join( - job_dict['COMOUT'], + output_JOB_file_format = os.path.join( + job_dict['job_num_work_dir'], job_dict['RUN']+'.'+job_dict['DATE'], model, job_dict['VERIF_CASE'], 'days6_10_avg_' +job_dict['VERIF_TYPE']+'_'+job_dict['job_name'] @@ -2779,17 +2641,8 @@ def check_model_files(job_dict): +'{valid?fmt=%Y%m%d}_' +'{valid?fmt=%H}0000V_' +'pairs.nc') - output_DATA_file_format = os.path.join( - verif_case_dir, 'METplus_output', - job_dict['RUN']+'.'+job_dict['DATE'], - model, job_dict['VERIF_CASE'], 'days6_10_avg_' - +job_dict['VERIF_TYPE']+'_'+job_dict['job_name'] - +'_init{init?fmt=%Y%m%d%H}_' - +'valid{valid_shift?fmt=%Y%m%d%H?shift=-120}' - +'to{valid?fmt=%Y%m%d%H}.nc' - ) - output_COMOUT_file_format = os.path.join( - job_dict['COMOUT'], + output_JOB_file_format = os.path.join( + job_dict['job_num_work_dir'], job_dict['RUN']+'.'+job_dict['DATE'], model, job_dict['VERIF_CASE'], 'days6_10_avg_' +job_dict['VERIF_TYPE']+'_'+job_dict['job_name'] @@ -2812,17 +2665,8 @@ def check_model_files(job_dict): +'{valid?fmt=%Y%m%d}_' +'{valid?fmt=%H}0000V_' +'pairs.nc') - output_DATA_file_format = os.path.join( - verif_case_dir, 'METplus_output', - job_dict['RUN']+'.'+job_dict['DATE'], - model, job_dict['VERIF_CASE'], 'weeks3_4_avg_' - +job_dict['VERIF_TYPE']+'_'+job_dict['job_name'] - +'_init{init?fmt=%Y%m%d%H}_' - +'valid{valid_shift?fmt=%Y%m%d%H?shift=-336}' - +'to{valid?fmt=%Y%m%d%H}.nc' - ) - output_COMOUT_file_format = os.path.join( - job_dict['COMOUT'], + output_JOB_file_format = os.path.join( + job_dict['job_num_work_dir'], job_dict['RUN']+'.'+job_dict['DATE'], model, job_dict['VERIF_CASE'], 'weeks3_4_avg_' +job_dict['VERIF_TYPE']+'_'+job_dict['job_name'] @@ -2845,17 +2689,8 @@ def check_model_files(job_dict): +'{valid?fmt=%Y%m%d}_' +'{valid?fmt=%H}0000V_' +'pairs.nc') - output_DATA_file_format = os.path.join( - verif_case_dir, 'METplus_output', - job_dict['RUN']+'.'+job_dict['DATE'], - model, job_dict['VERIF_CASE'], 'weeks3_4_avg_' - +job_dict['VERIF_TYPE']+'_'+job_dict['job_name'] - +'_init{init?fmt=%Y%m%d%H}_' - +'valid{valid_shift?fmt=%Y%m%d%H?shift=-336}' - +'to{valid?fmt=%Y%m%d%H}.nc' - ) - output_COMOUT_file_format = os.path.join( - job_dict['COMOUT'], + output_JOB_file_format = os.path.join( + job_dict['job_num_work_dir'], job_dict['RUN']+'.'+job_dict['DATE'], model, job_dict['VERIF_CASE'], 'weeks3_4_avg_' +job_dict['VERIF_TYPE']+'_'+job_dict['job_name'] @@ -2974,16 +2809,8 @@ def check_model_files(job_dict): } elif job_dict['JOB_GROUP'] == 'generate_stats': if job_dict['VERIF_CASE'] == 'grid2grid': - output_DATA_file_format = os.path.join( - verif_case_dir, 'METplus_output', - job_dict['RUN']+'.'+job_dict['DATE'], - model, job_dict['VERIF_CASE'], - 'grid_stat_'+job_dict['VERIF_TYPE']+'_' - +job_dict['job_name']+'_{lead?fmt=%2H}' - '0000L_{valid?fmt=%Y%m%d_%H%M%S}V.stat' - ) - output_COMOUT_file_format = os.path.join( - job_dict['COMOUT'], + output_JOB_file_format = os.path.join( + job_dict['job_num_work_dir'], job_dict['RUN']+'.'+job_dict['DATE'], model, job_dict['VERIF_CASE'], 'grid_stat_'+job_dict['VERIF_TYPE']+'_' @@ -3035,18 +2862,6 @@ def check_model_files(job_dict): +'valid{valid_shift?fmt=%Y%m%d%H?shift=-336}' +'to{valid?fmt=%Y%m%d%H}.nc' ) - #elif job_dict['VERIF_TYPE'] in ['sst', 'seaice'] \ - #and job_dict['job_name'] in ['WeeklyAvg_SST', - #'WeeklyAvg_Concentration']: - #input_file_format = os.path.join( - #verif_case_dir, 'METplus_output', - #job_dict['RUN']+'.'+job_dict['DATE'], - #model, job_dict['VERIF_CASE'], 'weekly_avg_' - #+job_dict['VERIF_TYPE']+'_'+job_dict['job_name'] - #+'_init{init?fmt=%Y%m%d%H}_' - #+'valid{valid_shift?fmt=%Y%m%d%H?shift=-168}' - #+'to{valid?fmt=%Y%m%d%H}.nc' - #) elif job_dict['VERIF_TYPE'] in ['sst', 'seaice'] \ and job_dict['job_name'] in ['MonthlyAvg_SST', 'MonthlyAvg_Concentration']: @@ -3168,17 +2983,8 @@ def check_model_files(job_dict): +'_init' +'{init?fmt=%Y%m%d%H}_' +'fhr{lead?fmt=%3H}.stat') - output_DATA_file_format = os.path.join( - verif_case_dir, 'METplus_output', - job_dict['RUN']+'.'+job_dict['DATE'], - model, job_dict['VERIF_CASE'], - 'stat_analysis_fcst'+job_dict['MODEL']+'_obsprepbufr_' - +job_dict['prepbufr']+'_'+job_dict['job_name'] - +'_SL1L2_valid{valid_shift?fmt=%Y%m%d%H?shift=-168}' - +'to{valid?fmt=%Y%m%d%H}.stat' - ) - output_COMOUT_file_format = os.path.join( - job_dict['COMOUT'], + output_JOB_file_format = os.path.join( + job_dict['job_num_work_dir'], job_dict['RUN']+'.'+job_dict['DATE'], model, job_dict['VERIF_CASE'], 'stat_analysis_fcst'+job_dict['MODEL']+'_obsprepbufr_' @@ -3202,17 +3008,8 @@ def check_model_files(job_dict): +'_init' +'{init?fmt=%Y%m%d%H}_' +'fhr{lead?fmt=%3H}.stat') - output_DATA_file_format = os.path.join( - verif_case_dir, 'METplus_output', - job_dict['RUN']+'.'+job_dict['DATE'], - model, job_dict['VERIF_CASE'], - 'stat_analysis_fcst'+job_dict['MODEL']+'_obsprepbufr_' - +job_dict['prepbufr']+'_'+job_dict['job_name'] - +'_SL1L2_valid{valid_shift?fmt=%Y%m%d%H?shift=-120}' - +'to{valid?fmt=%Y%m%d%H}.stat' - ) - output_COMOUT_file_format = os.path.join( - job_dict['COMOUT'], + output_JOB_file_format = os.path.join( + job_dict['job_num_work_dir'], job_dict['RUN']+'.'+job_dict['DATE'], model, job_dict['VERIF_CASE'], 'stat_analysis_fcst'+job_dict['MODEL']+'_obsprepbufr_' @@ -3236,17 +3033,8 @@ def check_model_files(job_dict): +'_init' +'{init?fmt=%Y%m%d%H}_' +'fhr{lead?fmt=%3H}.stat') - output_DATA_file_format = os.path.join( - verif_case_dir, 'METplus_output', - job_dict['RUN']+'.'+job_dict['DATE'], - model, job_dict['VERIF_CASE'], - 'stat_analysis_fcst'+job_dict['MODEL']+'_obsprepbufr_' - +job_dict['prepbufr']+'_'+job_dict['job_name'] - +'_SL1L2_valid{valid_shift?fmt=%Y%m%d%H?shift=-336}' - +'to{valid?fmt=%Y%m%d%H}.stat' - ) - output_COMOUT_file_format = os.path.join( - job_dict['COMOUT'], + output_JOB_file_format = os.path.join( + job_dict['job_num_work_dir'], job_dict['RUN']+'.'+job_dict['DATE'], model, job_dict['VERIF_CASE'], 'stat_analysis_fcst'+job_dict['MODEL']+'_obsprepbufr_' @@ -3269,17 +3057,8 @@ def check_model_files(job_dict): +'{valid?fmt=%Y%m%d}_' +'{valid?fmt=%H}0000V' +'.stat') - output_DATA_file_format = os.path.join( - verif_case_dir, 'METplus_output', - job_dict['RUN']+'.'+job_dict['DATE'], - model, job_dict['VERIF_CASE'], - 'stat_analysis_fcst'+job_dict['MODEL']+'_obsprepbufr_' - +job_dict['prepbufr']+'_'+job_dict['job_name'] - +'_SAL1L2_valid{valid_shift?fmt=%Y%m%d%H?shift=-168}' - +'to{valid?fmt=%Y%m%d%H}.stat' - ) - output_COMOUT_file_format = os.path.join( - job_dict['COMOUT'], + output_JOB_file_format = os.path.join( + job_dict['job_num_work_dir'], job_dict['RUN']+'.'+job_dict['DATE'], model, job_dict['VERIF_CASE'], 'stat_analysis_fcst'+job_dict['MODEL']+'_obsprepbufr_' @@ -3302,17 +3081,8 @@ def check_model_files(job_dict): +'{valid?fmt=%Y%m%d}_' +'{valid?fmt=%H}0000V' +'.stat') - output_DATA_file_format = os.path.join( - verif_case_dir, 'METplus_output', - job_dict['RUN']+'.'+job_dict['DATE'], - model, job_dict['VERIF_CASE'], - 'stat_analysis_fcst'+job_dict['MODEL']+'_obsprepbufr_' - +job_dict['prepbufr']+'_'+job_dict['job_name'] - +'_SAL1L2_valid{valid_shift?fmt=%Y%m%d%H?shift=-120}' - +'to{valid?fmt=%Y%m%d%H}.stat' - ) - output_COMOUT_file_format = os.path.join( - job_dict['COMOUT'], + output_JOB_file_format = os.path.join( + job_dict['job_num_work_dir'], job_dict['RUN']+'.'+job_dict['DATE'], model, job_dict['VERIF_CASE'], 'stat_analysis_fcst'+job_dict['MODEL']+'_obsprepbufr_' @@ -3335,17 +3105,8 @@ def check_model_files(job_dict): +'{valid?fmt=%Y%m%d}_' +'{valid?fmt=%H}0000V' +'.stat') - output_DATA_file_format = os.path.join( - verif_case_dir, 'METplus_output', - job_dict['RUN']+'.'+job_dict['DATE'], - model, job_dict['VERIF_CASE'], - 'stat_analysis_fcst'+job_dict['MODEL']+'_obsprepbufr_' - +job_dict['prepbufr']+'_'+job_dict['job_name'] - +'_SAL1L2_valid{valid_shift?fmt=%Y%m%d%H?shift=-336}' - +'to{valid?fmt=%Y%m%d%H}.stat' - ) - output_COMOUT_file_format = os.path.join( - job_dict['COMOUT'], + output_JOB_file_format = os.path.join( + job_dict['job_num_work_dir'], job_dict['RUN']+'.'+job_dict['DATE'], model, job_dict['VERIF_CASE'], 'stat_analysis_fcst'+job_dict['MODEL']+'_obsprepbufr_' @@ -3458,22 +3219,30 @@ def check_model_files(job_dict): ) input_fhr_list = copy.deepcopy(fhr_list) # Check output files - model_copy_output_DATA2COMOUT_list = [] + output_DATA_file_format = output_JOB_file_format.replace( + job_dict['job_num_work_dir'], + os.path.join(verif_case_dir, 'METplus_output') + ) + output_COMOUT_file_format = output_JOB_file_format.replace( + job_dict['job_num_work_dir'], job_dict['COMOUT'] + ) + model_copy_output_list = [] for fhr_key in list(fhr_check_output_dict.keys()): for fhr_fileN_key in list(fhr_check_output_dict[fhr_key].keys()): - fhr_fileN_DATA = format_filler( - output_DATA_file_format, + fhr_fileN_JOB = format_filler( + output_JOB_file_format, fhr_check_output_dict[fhr_key][fhr_fileN_key]['valid_date'], fhr_check_output_dict[fhr_key][fhr_fileN_key]['init_date'], fhr_check_output_dict[fhr_key][fhr_fileN_key]['forecast_hour'], {} ) - fhr_fileN_COMOUT = format_filler( - output_COMOUT_file_format, - fhr_check_output_dict[fhr_key][fhr_fileN_key]['valid_date'], - fhr_check_output_dict[fhr_key][fhr_fileN_key]['init_date'], - fhr_check_output_dict[fhr_key][fhr_fileN_key]['forecast_hour'], - {} + fhr_fileN_DATA = fhr_fileN_JOB.replace( + job_dict['job_num_work_dir'], + os.path.join(verif_case_dir, 'METplus_output') + ) + fhr_fileN_COMOUT = fhr_fileN_JOB.replace( + job_dict['job_num_work_dir'], + job_dict['COMOUT'] ) if os.path.exists(fhr_fileN_COMOUT): # Copy restart files from COMOUT to DATA dir @@ -3482,16 +3251,16 @@ def check_model_files(job_dict): if fhr_check_output_dict[fhr_key]\ [fhr_fileN_key]['forecast_hour'] \ in fhr_list: - if (fhr_fileN_DATA, fhr_fileN_COMOUT) \ - not in model_copy_output_DATA2COMOUT_list: - model_copy_output_DATA2COMOUT_list.append( - (fhr_fileN_DATA, fhr_fileN_COMOUT) + if (fhr_fileN_JOB, fhr_fileN_COMOUT) \ + not in model_copy_output_list: + model_copy_output_list.append( + (fhr_fileN_JOB, fhr_fileN_COMOUT) ) if len(fhr_list) != 0: model_files_exist = True else: model_files_exist = False - return model_files_exist, fhr_list, model_copy_output_DATA2COMOUT_list + return model_files_exist, fhr_list, model_copy_output_list def check_weekly_truth_files(job_dict): """! Check if obs files exist for weekly reformat and assemble step @@ -3503,8 +3272,7 @@ def check_weekly_truth_files(job_dict): Returns: truth_files_exist - if 80% of truth files exist or not (boolean) - truth_copy_output_DATA2COMOUT_list - list of files to copy - from DATA to COMOUT + truth_copy_output_list - list of files to copy """ valid_date_dt = datetime.datetime.strptime( job_dict['DATE']+job_dict['valid_hr_start'], @@ -3564,8 +3332,8 @@ def check_weekly_truth_files(job_dict): ) truth_input_file_list.append(prepbufr_file) nf+=1 - pb2nc_DATA_output_file_format = os.path.join( - verif_case_dir, 'METplus_output', + pb2nc_JOB_output_file_format = os.path.join( + job_dict['job_num_work_dir'], job_dict['RUN']+'.'+valid_date_dt.strftime('%Y%m%d'), 'prepbufr', job_dict['VERIF_CASE'], 'pb2nc_' +job_dict['VERIF_TYPE']+'_nam_valid' @@ -3580,8 +3348,8 @@ def check_weekly_truth_files(job_dict): ) nf = 0 while nf <= 14: - pb2nc_DATA_output = format_filler( - pb2nc_DATA_output_file_format, + pb2nc_JOB_output = format_filler( + pb2nc_JOB_output_file_format, (valid_date_dt-datetime.timedelta(hours=12*nf)), (valid_date_dt-datetime.timedelta(hours=12*nf)), ['anl'], {} @@ -3592,7 +3360,7 @@ def check_weekly_truth_files(job_dict): (valid_date_dt-datetime.timedelta(hours=12*nf)), ['anl'], {} ) - truth_output_file_list.append((pb2nc_DATA_output, + truth_output_file_list.append((pb2nc_JOB_output, pb2nc_COMOUT_output)) nf+=1 elif job_dict['JOB_GROUP'] == 'assemble_data': @@ -3617,11 +3385,11 @@ def check_weekly_truth_files(job_dict): truth_input_file_list.append(pb2nc_file) nf+=1 truth_output_files_exist_list = [] - truth_copy_output_DATA2COMOUT_list = truth_output_file_list + truth_copy_output_list = truth_output_file_list for truth_file_tuple in truth_output_file_list: if os.path.exists(truth_file_tuple[1]): truth_output_files_exist_list.append(True) - truth_copy_output_DATA2COMOUT_list.remove(truth_file_tuple) + truth_copy_output_list.remove(truth_file_tuple) else: truth_output_files_exist_list.append(False) if all(x == True for x in truth_output_files_exist_list) \ @@ -3639,7 +3407,7 @@ def check_weekly_truth_files(job_dict): truth_files_exist = True else: truth_files_exist = False - return truth_files_exist, truth_copy_output_DATA2COMOUT_list + return truth_files_exist, truth_copy_output_list def check_days6_10_truth_files(job_dict): """! Check if obs files exist for Days 6-10 reformat and assemble step @@ -3651,8 +3419,7 @@ def check_days6_10_truth_files(job_dict): Returns: truth_files_exist - if 80% of truth files exist or not (boolean) - truth_copy_output_DATA2COMOUT_list - list of files to copy - from DATA to COMOUT + truth_copy_output_list - list of files to copy """ valid_date_dt = datetime.datetime.strptime( job_dict['DATE']+job_dict['valid_hr_start'], @@ -3712,8 +3479,8 @@ def check_days6_10_truth_files(job_dict): ) truth_input_file_list.append(prepbufr_file) nf+=1 - pb2nc_DATA_output_file_format = os.path.join( - verif_case_dir, 'METplus_output', + pb2nc_JOB_output_file_format = os.path.join( + job_dict['job_num_work_dir'], job_dict['RUN']+'.'+valid_date_dt.strftime('%Y%m%d'), 'prepbufr', job_dict['VERIF_CASE'], 'pb2nc_' +job_dict['VERIF_TYPE']+'_nam_valid' @@ -3728,8 +3495,8 @@ def check_days6_10_truth_files(job_dict): ) nf = 0 while nf <= 10: - pb2nc_DATA_output = format_filler( - pb2nc_DATA_output_file_format, + pb2nc_JOB_output = format_filler( + pb2nc_JOB_output_file_format, (valid_date_dt-datetime.timedelta(hours=12*nf)), (valid_date_dt-datetime.timedelta(hours=12*nf)), ['anl'], {} @@ -3740,7 +3507,7 @@ def check_days6_10_truth_files(job_dict): (valid_date_dt-datetime.timedelta(hours=12*nf)), ['anl'], {} ) - truth_output_file_list.append((pb2nc_DATA_output, + truth_output_file_list.append((pb2nc_JOB_output, pb2nc_COMOUT_output)) nf+=1 elif job_dict['JOB_GROUP'] == 'assemble_data': @@ -3765,11 +3532,11 @@ def check_days6_10_truth_files(job_dict): truth_input_file_list.append(pb2nc_file) nf+=1 truth_output_files_exist_list = [] - truth_copy_output_DATA2COMOUT_list = truth_output_file_list + truth_copy_output_list = truth_output_file_list for truth_file_tuple in truth_output_file_list: if os.path.exists(truth_file_tuple[1]): truth_output_files_exist_list.append(True) - truth_copy_output_DATA2COMOUT_list.remove(truth_file_tuple) + truth_copy_output_list.remove(truth_file_tuple) else: truth_output_files_exist_list.append(False) if all(x == True for x in truth_output_files_exist_list) \ @@ -3787,7 +3554,7 @@ def check_days6_10_truth_files(job_dict): truth_files_exist = True else: truth_files_exist = False - return truth_files_exist, truth_copy_output_DATA2COMOUT_list + return truth_files_exist, truth_copy_output_list def check_weeks3_4_truth_files(job_dict): """! Check if obs files exist for Weeks 3-4 reformat and assemble step @@ -3799,8 +3566,7 @@ def check_weeks3_4_truth_files(job_dict): Returns: truth_files_exist - if 80% of truth files exist or not (boolean) - truth_copy_output_DATA2COMOUT_list - list of files to copy - from DATA to COMOUT + truth_copy_output_list - list of files to copy """ valid_date_dt = datetime.datetime.strptime( job_dict['DATE']+job_dict['valid_hr_start'], @@ -3811,6 +3577,7 @@ def check_weeks3_4_truth_files(job_dict): ) truth_input_file_list = [] truth_output_file_list = [] + truth_c_output_file_list = [] if job_dict['JOB_GROUP'] == 'reformat_data': if job_dict['VERIF_CASE'] == 'grid2grid': if job_dict['VERIF_TYPE'] == 'temp': @@ -3863,6 +3630,13 @@ def check_weeks3_4_truth_files(job_dict): ) truth_input_file_list.append(prepbufr_file) nf+=1 + pb2nc_JOB_output_file_format = os.path.join( + job_dict['job_num_work_dir'], + job_dict['RUN']+'.'+valid_date_dt.strftime('%Y%m%d'), + 'prepbufr', job_dict['VERIF_CASE'], 'pb2nc_' + +job_dict['VERIF_TYPE']+'_'+job_dict['prepbufr']+'_valid' + +'{valid?fmt=%Y%m%d%H}.nc' + ) pb2nc_DATA_output_file_format = os.path.join( verif_case_dir, 'METplus_output', job_dict['RUN']+'.'+valid_date_dt.strftime('%Y%m%d'), @@ -3879,6 +3653,12 @@ def check_weeks3_4_truth_files(job_dict): ) nf = 0 while nf <= 28: + pb2nc_JOB_output = format_filler( + pb2nc_JOB_output_file_format, + (valid_date_dt-datetime.timedelta(hours=12*nf)), + (valid_date_dt-datetime.timedelta(hours=12*nf)), + ['anl'], {} + ) pb2nc_DATA_output = format_filler( pb2nc_DATA_output_file_format, (valid_date_dt-datetime.timedelta(hours=12*nf)), @@ -3893,6 +3673,8 @@ def check_weeks3_4_truth_files(job_dict): ) truth_output_file_list.append((pb2nc_DATA_output, pb2nc_COMOUT_output)) + truth_c_output_file_list.append((pb2nc_JOB_output, + pb2nc_COMOUT_output)) nf+=1 elif job_dict['JOB_GROUP'] == 'assemble_data': if job_dict['VERIF_CASE'] == 'grid2obs': @@ -3916,13 +3698,15 @@ def check_weeks3_4_truth_files(job_dict): truth_input_file_list.append(pb2nc_file) nf+=1 truth_output_files_exist_list = [] - truth_copy_output_DATA2COMOUT_list = truth_output_file_list + if job_dict['VERIF_CASE'] == 'grid2grid': + truth_copy_output_list = truth_output_file_list + elif job_dict['VERIF_CASE'] == 'grid2obs': + truth_copy_output_list = truth_c_output_file_list for truth_file_tuple in truth_output_file_list: if os.path.exists(truth_file_tuple[1]): truth_output_files_exist_list.append(True) # Copy restart files from COMOUT to DATA dir copy_file(truth_file_tuple[1], truth_file_tuple[0]) - truth_copy_output_DATA2COMOUT_list.remove(truth_file_tuple) else: truth_output_files_exist_list.append(False) if all(x == True for x in truth_output_files_exist_list) \ @@ -3948,7 +3732,7 @@ def check_weeks3_4_truth_files(job_dict): truth_files_exist = True else: truth_files_exist = False - return truth_files_exist, truth_copy_output_DATA2COMOUT_list + return truth_files_exist, truth_copy_output_list def check_truth_files(job_dict): """! From d43a2075f6ed4106f37ae3457fec089fcfd301aa Mon Sep 17 00:00:00 2001 From: PerryShafran-NOAA <62255233+PerryShafran-NOAA@users.noreply.github.com> Date: Tue, 19 Nov 2024 13:52:02 -0500 Subject: [PATCH 2/6] Mesoscale NAM/RAP Fixes and Additions - no restart (#603) * Updating dev driver scripts * Updating jobs scripts * Updating parm files * Updating exevs scripts * First update of ush directory * Second update of ush directory * Making updates from Wei * Updating ecf with Wei's change * Correcting ncpus * Updating stats ecf for reduction in runtime * Updating nproc * Updating ecf and ush files * Remove unneeded lines * To fix RESTART_DIR error * Fix COMOUTsmall * Updating memory resources for ecf and dev drivers * Removing remaining log file prints * Updating headline plot resources * Removing testing directories --------- Co-authored-by: perry shafran --- .../jevs_mesoscale_grid2obs_plots.sh | 4 +- .../jevs_mesoscale_headline_plots.sh | 4 +- .../mesoscale/jevs_mesoscale_precip_plots.sh | 6 +- .../jevs_mesoscale_snowfall_plots.sh | 4 +- .../jevs_mesoscale_nam_grid2obs_stats.sh | 4 +- .../jevs_mesoscale_rap_grid2obs_stats.sh | 4 +- .../jevs_mesoscale_grid2obs_plots.ecf | 2 +- .../jevs_mesoscale_headline_plots.ecf | 4 +- .../mesoscale/jevs_mesoscale_precip_plots.ecf | 2 +- .../jevs_mesoscale_snowfall_plots.ecf | 2 +- .../jevs_mesoscale_nam_grid2obs_stats.ecf | 4 +- .../jevs_mesoscale_rap_grid2obs_stats.ecf | 4 +- jobs/JEVS_MESOSCALE_PLOTS | 16 +- jobs/JEVS_MESOSCALE_STATS | 8 + ...vs.prod.stats.mesoscale.atmos.grid2obs.nam | 4 - ...vs.prod.stats.mesoscale.atmos.grid2obs.rap | 4 - .../mesoscale/grid2obs/PB2NC_obsMETAR.conf | 10 +- .../mesoscale/grid2obs/PB2NC_obsRAOB.conf | 10 +- .../PointStat_fcstMESOSCALE_obsMETAR.conf | 6 +- ...ointStat_fcstMESOSCALE_obsMETAR_PTYPE.conf | 6 +- ...PointStat_fcstMESOSCALE_obsMETAR_VAR2.conf | 6 +- ...ntStat_fcstMESOSCALE_obsNDAS_PrepBufr.conf | 6 +- .../PointStat_fcstMESOSCALE_obsRAOB.conf | 6 +- .../PointStat_fcstMESOSCALE_obsRAOB_VAR2.conf | 6 +- .../RegridDataPlane_fcstMESOSCALE_PTYPE.conf | 4 +- ...tAnalysis_fcstMESOSCALE_GatherByCycle.conf | 6 +- ...tatAnalysis_fcstMESOSCALE_GatherByDay.conf | 8 +- ...is_fcstMESOSCALE_obsMETAR_GatherByDay.conf | 6 +- ...sis_fcstMESOSCALE_obsNDAS_GatherByDay.conf | 2 - ...sis_fcstMESOSCALE_obsRAOB_GatherByDay.conf | 6 +- .../exevs_mesoscale_grid2obs_plots.sh | 2 +- .../exevs_mesoscale_nam_grid2obs_stats.sh | 51 ++-- .../exevs_mesoscale_nam_precip_stats.sh | 5 - .../exevs_mesoscale_nam_snowfall_stats.sh | 4 - .../exevs_mesoscale_rap_grid2obs_stats.sh | 59 ++-- .../exevs_mesoscale_rap_precip_stats.sh | 4 - .../exevs_mesoscale_rap_snowfall_stats.sh | 4 - ush/mesoscale/mesoscale_check_input_data.py | 42 +-- ush/mesoscale/mesoscale_check_settings.py | 18 +- ush/mesoscale/mesoscale_create_output_dirs.py | 23 +- ...oscale_stats_grid2obs_create_job_script.py | 4 - ...cale_stats_grid2obs_preprocess_prepbufr.py | 47 +++ .../mesoscale_stats_grid2obs_var_defs.py | 284 ++++++------------ ush/mesoscale/mesoscale_util.py | 135 ++++++++- ush/mesoscale/prune_stat_files.py | 4 +- 45 files changed, 394 insertions(+), 456 deletions(-) create mode 100644 ush/mesoscale/mesoscale_stats_grid2obs_preprocess_prepbufr.py diff --git a/dev/drivers/scripts/plots/mesoscale/jevs_mesoscale_grid2obs_plots.sh b/dev/drivers/scripts/plots/mesoscale/jevs_mesoscale_grid2obs_plots.sh index a44e979d7..09c13db5c 100755 --- a/dev/drivers/scripts/plots/mesoscale/jevs_mesoscale_grid2obs_plots.sh +++ b/dev/drivers/scripts/plots/mesoscale/jevs_mesoscale_grid2obs_plots.sh @@ -4,7 +4,7 @@ #PBS -q dev #PBS -A VERF-DEV #PBS -l walltime=4:00:00 -#PBS -l place=vscatter:exclhost,select=12:ncpus=128:mem=150GB +#PBS -l place=vscatter:exclhost,select=12:ncpus=128:mem=500GB #PBS -l debug=true set -x @@ -23,7 +23,7 @@ export job=${PBS_JOBNAME:-jevs_mesoscale_grid2obs_plots} export jobid=$job.${PBS_JOBID:-$$} export SITE=$(cat /etc/cluster_name) export USE_CFP=YES -export nproc=128 +export nproc=64 export evs_run_mode="production" # General Verification Settings diff --git a/dev/drivers/scripts/plots/mesoscale/jevs_mesoscale_headline_plots.sh b/dev/drivers/scripts/plots/mesoscale/jevs_mesoscale_headline_plots.sh index 75dfbfdb1..73d3c5915 100755 --- a/dev/drivers/scripts/plots/mesoscale/jevs_mesoscale_headline_plots.sh +++ b/dev/drivers/scripts/plots/mesoscale/jevs_mesoscale_headline_plots.sh @@ -4,7 +4,7 @@ #PBS -q dev #PBS -A VERF-DEV #PBS -l walltime=00:30:00 -#PBS -l place=vscatter:exclhost,select=1:ncpus=128:mem=150GB:ompthreads=1 +#PBS -l place=vscatter:exclhost,select=1:ncpus=16:mem=20GB:ompthreads=1 #PBS -l debug=true set -x @@ -22,7 +22,7 @@ export job=${PBS_JOBNAME:-jevs_mesoscale_headline_plots} export jobid=$job.${PBS_JOBID:-$$} export SITE=$(cat /etc/cluster_name) export USE_CFP=YES -export nproc=128 +export nproc=16 export evs_run_mode="production" # General Verification Settings diff --git a/dev/drivers/scripts/plots/mesoscale/jevs_mesoscale_precip_plots.sh b/dev/drivers/scripts/plots/mesoscale/jevs_mesoscale_precip_plots.sh index bf6bd45d0..e1ffefadb 100755 --- a/dev/drivers/scripts/plots/mesoscale/jevs_mesoscale_precip_plots.sh +++ b/dev/drivers/scripts/plots/mesoscale/jevs_mesoscale_precip_plots.sh @@ -3,8 +3,8 @@ #PBS -S /bin/bash #PBS -q dev #PBS -A VERF-DEV -#PBS -l walltime=04:00:00 -#PBS -l place=vscatter:exclhost,select=12:ncpus=128:mem=150GB +#PBS -l walltime=05:00:00 +#PBS -l place=vscatter:exclhost,select=12:ncpus=128:mem=500GB #PBS -l debug=true set -x @@ -49,7 +49,7 @@ export PYTHONPATH=$HOMEevs/ush/$COMPONENT:$PYTHONPATH # Developer Settings export COMIN=/lfs/h2/emc/vpppg/noscrub/${USER}/$NET/$evs_ver_2d -export DATAROOT=/lfs/h2/emc/ptmp/${USER}/evs_test/$envir/tmp +export DATAROOT=/lfs/h2/emc/stmp/${USER}/evs_test/$envir/tmp export COMOUT=/lfs/h2/emc/ptmp/${USER}/$NET/$evs_ver_2d/$STEP/$COMPONENT export vhr=${vhr:-${vhr}} diff --git a/dev/drivers/scripts/plots/mesoscale/jevs_mesoscale_snowfall_plots.sh b/dev/drivers/scripts/plots/mesoscale/jevs_mesoscale_snowfall_plots.sh index 6eb7abc0a..adb532fe7 100755 --- a/dev/drivers/scripts/plots/mesoscale/jevs_mesoscale_snowfall_plots.sh +++ b/dev/drivers/scripts/plots/mesoscale/jevs_mesoscale_snowfall_plots.sh @@ -3,8 +3,8 @@ #PBS -S /bin/bash #PBS -q dev #PBS -A VERF-DEV -#PBS -l walltime=3:00:00 -#PBS -l place=vscatter:exclhost,select=12:ncpus=128:mem=150GB +#PBS -l walltime=6:00:00 +#PBS -l place=vscatter:exclhost,select=12:ncpus=128:mem=500GB #PBS -l debug=true set -x diff --git a/dev/drivers/scripts/stats/mesoscale/jevs_mesoscale_nam_grid2obs_stats.sh b/dev/drivers/scripts/stats/mesoscale/jevs_mesoscale_nam_grid2obs_stats.sh index 91a332a6b..fae6d9867 100755 --- a/dev/drivers/scripts/stats/mesoscale/jevs_mesoscale_nam_grid2obs_stats.sh +++ b/dev/drivers/scripts/stats/mesoscale/jevs_mesoscale_nam_grid2obs_stats.sh @@ -3,8 +3,8 @@ #PBS -S /bin/bash #PBS -q "dev" #PBS -A VERF-DEV -#PBS -l walltime=4:59:00 -#PBS -l place=vscatter:exclhost,select=3:ncpus=128:ompthreads=1:mem=150GB +#PBS -l walltime=01:00:00 +#PBS -l place=vscatter:exclhost,select=3:ncpus=128:ompthreads=1:mem=500GB #PBS -l debug=true set -x diff --git a/dev/drivers/scripts/stats/mesoscale/jevs_mesoscale_rap_grid2obs_stats.sh b/dev/drivers/scripts/stats/mesoscale/jevs_mesoscale_rap_grid2obs_stats.sh index a81befd5f..9607ac7c3 100755 --- a/dev/drivers/scripts/stats/mesoscale/jevs_mesoscale_rap_grid2obs_stats.sh +++ b/dev/drivers/scripts/stats/mesoscale/jevs_mesoscale_rap_grid2obs_stats.sh @@ -3,8 +3,8 @@ #PBS -S /bin/bash #PBS -q "dev" #PBS -A VERF-DEV -#PBS -l walltime=7:00:00 -#PBS -l place=vscatter:exclhost,select=3:ncpus=128:ompthreads=1:mem=150GB +#PBS -l walltime=01:30:00 +#PBS -l place=vscatter:exclhost,select=3:ncpus=128:ompthreads=1:mem=500GB #PBS -l debug=true set -x diff --git a/ecf/scripts/plots/mesoscale/jevs_mesoscale_grid2obs_plots.ecf b/ecf/scripts/plots/mesoscale/jevs_mesoscale_grid2obs_plots.ecf index a66f0e6d9..3d7565d1f 100755 --- a/ecf/scripts/plots/mesoscale/jevs_mesoscale_grid2obs_plots.ecf +++ b/ecf/scripts/plots/mesoscale/jevs_mesoscale_grid2obs_plots.ecf @@ -50,7 +50,7 @@ if [ -n "%VHR:%" ]; then else export vhr=00 fi -export nproc=128 +export nproc=64 export evs_run_mode="production" export USE_CFP=YES export NET=evs diff --git a/ecf/scripts/plots/mesoscale/jevs_mesoscale_headline_plots.ecf b/ecf/scripts/plots/mesoscale/jevs_mesoscale_headline_plots.ecf index a91c7ef3b..a1805b1ba 100755 --- a/ecf/scripts/plots/mesoscale/jevs_mesoscale_headline_plots.ecf +++ b/ecf/scripts/plots/mesoscale/jevs_mesoscale_headline_plots.ecf @@ -4,7 +4,7 @@ #PBS -q %QUEUE% #PBS -A %PROJ%-%PROJENVIR% #PBS -l walltime=00:15:00 -#PBS -l place=vscatter:exclhost,select=1:ncpus=128:ompthreads=1:mem=500GB +#PBS -l place=vscatter:exclhost,select=1:ncpus=16:ompthreads=1:mem=20GB #PBS -l debug=true export model=evs @@ -50,7 +50,7 @@ if [ -n "%VHR:%" ]; then else export vhr=00 fi -export nproc=128 +export nproc=16 export USE_CFP=YES export NET=evs export RUN=atmos diff --git a/ecf/scripts/plots/mesoscale/jevs_mesoscale_precip_plots.ecf b/ecf/scripts/plots/mesoscale/jevs_mesoscale_precip_plots.ecf index 258f0efd7..e4583127b 100755 --- a/ecf/scripts/plots/mesoscale/jevs_mesoscale_precip_plots.ecf +++ b/ecf/scripts/plots/mesoscale/jevs_mesoscale_precip_plots.ecf @@ -3,7 +3,7 @@ #PBS -S /bin/bash #PBS -q %QUEUE% #PBS -A %PROJ%-%PROJENVIR% -#PBS -l walltime=06:50:00 +#PBS -l walltime=05:00:00 #PBS -l place=vscatter:exclhost,select=12:ncpus=128:mem=500GB #PBS -l debug=true diff --git a/ecf/scripts/plots/mesoscale/jevs_mesoscale_snowfall_plots.ecf b/ecf/scripts/plots/mesoscale/jevs_mesoscale_snowfall_plots.ecf index d6947f2df..80c9b0ce6 100755 --- a/ecf/scripts/plots/mesoscale/jevs_mesoscale_snowfall_plots.ecf +++ b/ecf/scripts/plots/mesoscale/jevs_mesoscale_snowfall_plots.ecf @@ -3,7 +3,7 @@ #PBS -S /bin/bash #PBS -q %QUEUE% #PBS -A %PROJ%-%PROJENVIR% -#PBS -l walltime=10:15:00 +#PBS -l walltime=06:00:00 #PBS -l place=vscatter:exclhost,select=12:ncpus=128:mem=500GB #PBS -l debug=true diff --git a/ecf/scripts/stats/mesoscale/jevs_mesoscale_nam_grid2obs_stats.ecf b/ecf/scripts/stats/mesoscale/jevs_mesoscale_nam_grid2obs_stats.ecf index 7030e7023..6c5d73512 100755 --- a/ecf/scripts/stats/mesoscale/jevs_mesoscale_nam_grid2obs_stats.ecf +++ b/ecf/scripts/stats/mesoscale/jevs_mesoscale_nam_grid2obs_stats.ecf @@ -3,8 +3,7 @@ #PBS -S /bin/bash #PBS -q %QUEUE% #PBS -A %PROJ%-%PROJENVIR% -#PBS -l walltime=04:15:00 -##PBS -l walltime=02:10:00 +#PBS -l walltime=01:00:00 #PBS -l place=vscatter:exclhost,select=3:ncpus=128:ompthreads=1:mem=500GB #PBS -l debug=true @@ -38,6 +37,7 @@ module load grib_util/${grib_util_ver} module load wgrib2/${wgrib2_ver} module load met/${met_ver} module load metplus/${metplus_ver} +module load bufr/${bufr_ver} module list ############################################################ diff --git a/ecf/scripts/stats/mesoscale/jevs_mesoscale_rap_grid2obs_stats.ecf b/ecf/scripts/stats/mesoscale/jevs_mesoscale_rap_grid2obs_stats.ecf index 820b7b561..88445b645 100755 --- a/ecf/scripts/stats/mesoscale/jevs_mesoscale_rap_grid2obs_stats.ecf +++ b/ecf/scripts/stats/mesoscale/jevs_mesoscale_rap_grid2obs_stats.ecf @@ -3,8 +3,7 @@ #PBS -S /bin/bash #PBS -q %QUEUE% #PBS -A %PROJ%-%PROJENVIR% -#PBS -l walltime=08:00:00 -##PBS -l walltime=04:15:00 +#PBS -l walltime=01:30:00 #PBS -l place=vscatter:exclhost,select=3:ncpus=128:ompthreads=1:mem=500GB #PBS -l debug=true @@ -38,6 +37,7 @@ module load grib_util/${grib_util_ver} module load wgrib2/${wgrib2_ver} module load met/${met_ver} module load metplus/${metplus_ver} +module load bufr/${bufr_ver} module list ############################################################ diff --git a/jobs/JEVS_MESOSCALE_PLOTS b/jobs/JEVS_MESOSCALE_PLOTS index 36065fd6c..ec9a6f425 100755 --- a/jobs/JEVS_MESOSCALE_PLOTS +++ b/jobs/JEVS_MESOSCALE_PLOTS @@ -19,6 +19,14 @@ export DATA=${DATA:-${DATAROOT:?}/${jobid:?}} mkdir -p $DATA cd $DATA +################################# +# Define NET/RUN variables +################################# +export NET=${NET:-evs} +export STEP=${STEP:-plots} +export COMPONENT=${COMPONENT:-mesoscale} +export RUN=${RUN:-atmos} + #################################### # Set EVS directories @@ -80,14 +88,6 @@ else fi -#################################### -# Define NET/RUN variables: -#################################### -export NET=${NET:-evs} -export STEP=${STEP:-stats} -export COMPONENT=${COMPONENT:-mesoscale} -export RUN=${RUN:-atmos} - if [ $RUN = atmos ] ; then export INITDATE=${INITDATE:-$PDYm2} fi diff --git a/jobs/JEVS_MESOSCALE_STATS b/jobs/JEVS_MESOSCALE_STATS index 6f40ae837..48c9b9116 100755 --- a/jobs/JEVS_MESOSCALE_STATS +++ b/jobs/JEVS_MESOSCALE_STATS @@ -41,6 +41,14 @@ export DATA=${DATA:-${DATAROOT:?}/${jobid:?}} mkdir -p $DATA cd $DATA +################################ +# Define NET/RUN variables +################################ +export NET=${NET:-evs} +export STEP=${STEP:-stats} +export COMPONENT=${COMPONENT:-mesoscale} +export RUN=${RUN:-atmos} + export machine=${machine:-WCOSS2} export MET_PLUS_PATH=${MET_PLUS_PATH:-$METPLUS_PATH} export MET_PATH=$MET_ROOT diff --git a/parm/evs_config/mesoscale/config.evs.prod.stats.mesoscale.atmos.grid2obs.nam b/parm/evs_config/mesoscale/config.evs.prod.stats.mesoscale.atmos.grid2obs.nam index 3d14b98f6..585c1ea32 100644 --- a/parm/evs_config/mesoscale/config.evs.prod.stats.mesoscale.atmos.grid2obs.nam +++ b/parm/evs_config/mesoscale/config.evs.prod.stats.mesoscale.atmos.grid2obs.nam @@ -15,10 +15,6 @@ echo "BEGIN: $(basename ${BASH_SOURCE[0]})" export MET_PLUS_OUT=${DATA}/${VERIF_CASE}/METplus_output export MET_CONFIG_OVERRIDES="" -# Logging -export METPLUS_VERBOSITY="DEBUG" -export MET_VERBOSITY="5" - # Time Settings export FHR_START=0 export FHR_INCR=6 diff --git a/parm/evs_config/mesoscale/config.evs.prod.stats.mesoscale.atmos.grid2obs.rap b/parm/evs_config/mesoscale/config.evs.prod.stats.mesoscale.atmos.grid2obs.rap index d4cdb53eb..70cdfadf0 100644 --- a/parm/evs_config/mesoscale/config.evs.prod.stats.mesoscale.atmos.grid2obs.rap +++ b/parm/evs_config/mesoscale/config.evs.prod.stats.mesoscale.atmos.grid2obs.rap @@ -15,10 +15,6 @@ echo "BEGIN: $(basename ${BASH_SOURCE[0]})" export MET_PLUS_OUT=${DATA}/${VERIF_CASE}/METplus_output export MET_CONFIG_OVERRIDES="" -# Logging -export METPLUS_VERBOSITY="DEBUG" -export MET_VERBOSITY="5" - # Time Settings export FHR_START=0 export FHR_INCR=1 diff --git a/parm/metplus_config/stats/mesoscale/grid2obs/PB2NC_obsMETAR.conf b/parm/metplus_config/stats/mesoscale/grid2obs/PB2NC_obsMETAR.conf index a890e3dcd..c1c328781 100755 --- a/parm/metplus_config/stats/mesoscale/grid2obs/PB2NC_obsMETAR.conf +++ b/parm/metplus_config/stats/mesoscale/grid2obs/PB2NC_obsMETAR.conf @@ -4,7 +4,7 @@ MET_INSTALL_DIR = {ENV[MET_PATH]} MET_BIN_DIR = {ENV[MET_PATH]}/bin METPLUS_PARM_BASE = {ENV[MET_PLUS_CONF]} -INPUT_BASE = {ENV[COMINobs]} +INPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}/data/{ENV[VERIF_TYPE]}/prepbufr OUTPUT_BASE = {ENV[MET_PLUS_OUT]}/{ENV[VERIF_TYPE]}/{ENV[NEST]}/pb2nc CONFIG_DIR = {ENV[MET_CONFIG]} LOG_DIR = {OUTPUT_BASE}/logs @@ -16,12 +16,10 @@ PB2NC_INPUT_DIR = {INPUT_BASE} PB2NC_OUTPUT_DIR = {OUTPUT_BASE} [filename_templates] -PB2NC_INPUT_TEMPLATE = nam.{da_init?fmt=%Y%m%d}/nam.t{cycle?fmt=%2H}z.prepbufr.tm{offset?fmt=%2H} +PB2NC_INPUT_TEMPLATE = nam.t{cycle?fmt=%2H}z.prepbufr.tm{offset?fmt=%2H} PB2NC_OUTPUT_TEMPLATE = prepbufr.{MODEL1}.{NEST}.{valid?fmt=%Y%m%d%H}.nc [config] -LOG_LEVEL = {ENV[METPLUS_VERBOSITY]} -LOG_MET_VERBOSITY = {ENV[MET_VERBOSITY]} LOG_PB2NC_VERBOSITY = {LOG_MET_VERBOSITY} LOG_TIMESTAMP_TEMPLATE = %Y%m%d%H%M%S LOG_TIMESTAMP_USE_DATATIME = no @@ -62,7 +60,7 @@ PB2NC_TIME_SUMMARY_VALID_FREQ = 0 PB2NC_TIME_SUMMARY_VALID_THRESH = 0.0 PB2NC_MET_CONFIG_OVERRIDES = obs_prepbufr_map = [ { key = "ZOB"; val = "HGT"; }, { key = "POB"; val = "PRES"; }, { key = "QOB"; val = "SPFH"; }, { key = "TOB"; val = "TMP"; }, { key = "UOB"; val = "UGRD"; }, { key = "VOB"; val = "VGRD"; }, { key = "D_WDIR"; val = "WDIR"; }, { key = "D_WIND"; val = "WIND"; }, { key = "D_RH"; val = "RH"; }, { key = "D_MIXR"; val = "MIXR"; }, { key = "D_PBL"; val = "HPBL"; }, { key = "D_CAPE"; val = "CAPE"; }, { key = "TDO"; val = "DPT"; }, { key = "PMO"; val = "PRMSL"; }, { key = "TOCC"; val = "TCDC"; }, { key = "HOVI"; val = "VIS"; }, { key = "CEILING"; val = "CEILING"; }, { key = "MXGS"; val = "GUST";}, { key = "D_MLCAPE"; val = "MLCAPE"; } ]; PB2NC_OFFSETS = 6, 3 -PB2NC_SKIP_TIMES = +PB2NC_SKIP_VALID_TIMES = PB2NC_MANDATORY = True LOOP_BY = VALID @@ -71,7 +69,7 @@ VALID_TIME_FMT = %Y%m%d%H VALID_BEG = {ENV[VDATE]}{ENV[VHOUR]} VALID_END = {ENV[VDATE]}{ENV[VHOUR]} VALID_INCREMENT = 24H -SKIP_TIMES = +SKIP_VALID_TIMES = LEAD_SEQ = 0 OMP_NUM_THREADS = 1 diff --git a/parm/metplus_config/stats/mesoscale/grid2obs/PB2NC_obsRAOB.conf b/parm/metplus_config/stats/mesoscale/grid2obs/PB2NC_obsRAOB.conf index 70f3da205..3b310ecd9 100755 --- a/parm/metplus_config/stats/mesoscale/grid2obs/PB2NC_obsRAOB.conf +++ b/parm/metplus_config/stats/mesoscale/grid2obs/PB2NC_obsRAOB.conf @@ -4,7 +4,7 @@ MET_INSTALL_DIR = {ENV[MET_PATH]} MET_BIN_DIR = {ENV[MET_PATH]}/bin METPLUS_PARM_BASE = {ENV[MET_PLUS_CONF]} -INPUT_BASE = {ENV[COMINobs]} +INPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}/data/{ENV[VERIF_TYPE]}/prepbufr OUTPUT_BASE = {ENV[MET_PLUS_OUT]}/{ENV[VERIF_TYPE]}/{ENV[NEST]}/pb2nc CONFIG_DIR = {ENV[MET_CONFIG]} LOG_DIR = {OUTPUT_BASE}/logs @@ -16,12 +16,10 @@ PB2NC_INPUT_DIR = {INPUT_BASE} PB2NC_OUTPUT_DIR = {OUTPUT_BASE} [filename_templates] -PB2NC_INPUT_TEMPLATE = gdas.{valid?fmt=%Y%m%d}/{valid?fmt=%2H}/atmos/gdas.t{valid?fmt=%2H}z.prepbufr +PB2NC_INPUT_TEMPLATE = gdas.t{valid?fmt=%2H}z.prepbufr PB2NC_OUTPUT_TEMPLATE = prepbufr.{MODEL1}.{NEST}.{valid?fmt=%Y%m%d%H}.nc [config] -LOG_LEVEL = {ENV[METPLUS_VERBOSITY]} -LOG_MET_VERBOSITY = {ENV[MET_VERBOSITY]} LOG_PB2NC_VERBOSITY = {LOG_MET_VERBOSITY} LOG_TIMESTAMP_TEMPLATE = %Y%m%d%H%M%S LOG_TIMESTAMP_USE_DATATIME = no @@ -62,7 +60,7 @@ PB2NC_TIME_SUMMARY_VALID_FREQ = 0 PB2NC_TIME_SUMMARY_VALID_THRESH = 0.0 PB2NC_MET_CONFIG_OVERRIDES = obs_prepbufr_map = [ { key = "ZOB"; val = "HGT"; }, { key = "POB"; val = "PRES"; }, { key = "QOB"; val = "SPFH"; }, { key = "TOB"; val = "TMP"; }, { key = "UOB"; val = "UGRD"; }, { key = "VOB"; val = "VGRD"; }, { key = "D_WDIR"; val = "WDIR"; }, { key = "D_WIND"; val = "WIND"; }, { key = "D_RH"; val = "RH"; }, { key = "D_MIXR"; val = "MIXR"; }, { key = "D_PBL"; val = "HPBL"; }, { key = "D_CAPE"; val = "CAPE"; }, { key = "TDO"; val = "DPT"; }, { key = "PMO"; val = "PRMSL"; }, { key = "TOCC"; val = "TCDC"; }, { key = "HOVI"; val = "VIS"; }, { key = "CEILING"; val = "CEILING"; }, { key = "MXGS"; val = "GUST";}, { key = "D_MLCAPE"; val = "MLCAPE"; } ]; PB2NC_OFFSETS = 6, 5, 4, 3, 2, 1 -PB2NC_SKIP_TIMES = +PB2NC_SKIP_VALID_TIMES = PB2NC_MANDATORY = True LOOP_BY = VALID @@ -71,7 +69,7 @@ VALID_TIME_FMT = %Y%m%d%H VALID_BEG = {ENV[VDATE]}{ENV[VHOUR]} VALID_END = {ENV[VDATE]}{ENV[VHOUR]} VALID_INCREMENT = 24H -SKIP_TIMES = +SKIP_VALID_TIMES = LEAD_SEQ = 0 OMP_NUM_THREADS = 1 diff --git a/parm/metplus_config/stats/mesoscale/grid2obs/PointStat_fcstMESOSCALE_obsMETAR.conf b/parm/metplus_config/stats/mesoscale/grid2obs/PointStat_fcstMESOSCALE_obsMETAR.conf index 4a05afde4..2b980aabb 100755 --- a/parm/metplus_config/stats/mesoscale/grid2obs/PointStat_fcstMESOSCALE_obsMETAR.conf +++ b/parm/metplus_config/stats/mesoscale/grid2obs/PointStat_fcstMESOSCALE_obsMETAR.conf @@ -25,8 +25,6 @@ POINT_STAT_OUTPUT_TEMPLATE = {MODEL}.{valid?fmt=%Y%m%d} POINT_STAT_VERIFICATION_MASK_TEMPLATE = {ENV[MASK_POLY_LIST]} [config] -LOG_LEVEL = {ENV[METPLUS_VERBOSITY]} -LOG_MET_VERBOSITY = {ENV[MET_VERBOSITY]} LOG_POINT_STAT_VERBOSITY = {LOG_MET_VERBOSITY} LOG_TIMESTAMP_TEMPLATE = %Y%m%d%H%M%S LOG_TIMESTAMP_USE_DATATIME = no @@ -50,8 +48,8 @@ VALID_TIME_FMT = %Y%m%d%H VALID_BEG = {ENV[VDATE]}{ENV[VHOUR]} VALID_END = {ENV[VDATE]}{ENV[VHOUR]} VALID_INCREMENT = 24H -POINT_STAT_SKIP_TIMES = -SKIP_TIMES = +POINT_STAT_SKIP_VALID_TIMES = +SKIP_VALID_TIMES = LEAD_SEQ = begin_end_incr({ENV[FHR_START]},{ENV[FHR_END]},{ENV[FHR_INCR]}) # Variables diff --git a/parm/metplus_config/stats/mesoscale/grid2obs/PointStat_fcstMESOSCALE_obsMETAR_PTYPE.conf b/parm/metplus_config/stats/mesoscale/grid2obs/PointStat_fcstMESOSCALE_obsMETAR_PTYPE.conf index 5c4528e91..9a4e8901c 100755 --- a/parm/metplus_config/stats/mesoscale/grid2obs/PointStat_fcstMESOSCALE_obsMETAR_PTYPE.conf +++ b/parm/metplus_config/stats/mesoscale/grid2obs/PointStat_fcstMESOSCALE_obsMETAR_PTYPE.conf @@ -26,8 +26,6 @@ POINT_STAT_OUTPUT_TEMPLATE = {MODEL}.{valid?fmt=%Y%m%d} POINT_STAT_VERIFICATION_MASK_TEMPLATE = {ENV[MASK_POLY_LIST]} [config] -LOG_LEVEL = {ENV[METPLUS_VERBOSITY]} -LOG_MET_VERBOSITY = {ENV[MET_VERBOSITY]} LOG_POINT_STAT_VERBOSITY = {LOG_MET_VERBOSITY} LOG_TIMESTAMP_TEMPLATE = %Y%m%d%H%M%S LOG_TIMESTAMP_USE_DATATIME = no @@ -51,8 +49,8 @@ VALID_TIME_FMT = %Y%m%d%H VALID_BEG = {ENV[VDATE]}{ENV[VHOUR]} VALID_END = {ENV[VDATE]}{ENV[VHOUR]} VALID_INCREMENT = 24H -POINT_STAT_SKIP_TIMES = -SKIP_TIMES = +POINT_STAT_SKIP_VALID_TIMES = +SKIP_VALID_TIMES = LEAD_SEQ = {ENV[FHR]} # Variables diff --git a/parm/metplus_config/stats/mesoscale/grid2obs/PointStat_fcstMESOSCALE_obsMETAR_VAR2.conf b/parm/metplus_config/stats/mesoscale/grid2obs/PointStat_fcstMESOSCALE_obsMETAR_VAR2.conf index 175538d66..9e3f6a7c9 100755 --- a/parm/metplus_config/stats/mesoscale/grid2obs/PointStat_fcstMESOSCALE_obsMETAR_VAR2.conf +++ b/parm/metplus_config/stats/mesoscale/grid2obs/PointStat_fcstMESOSCALE_obsMETAR_VAR2.conf @@ -25,8 +25,6 @@ POINT_STAT_OUTPUT_TEMPLATE = {MODEL}.{valid?fmt=%Y%m%d} POINT_STAT_VERIFICATION_MASK_TEMPLATE = {ENV[MASK_POLY_LIST]} [config] -LOG_LEVEL = {ENV[METPLUS_VERBOSITY]} -LOG_MET_VERBOSITY = {ENV[MET_VERBOSITY]} LOG_POINT_STAT_VERBOSITY = {LOG_MET_VERBOSITY} LOG_TIMESTAMP_TEMPLATE = %Y%m%d%H%M%S LOG_TIMESTAMP_USE_DATATIME = no @@ -50,8 +48,8 @@ VALID_TIME_FMT = %Y%m%d%H VALID_BEG = {ENV[VDATE]}{ENV[VHOUR]} VALID_END = {ENV[VDATE]}{ENV[VHOUR]} VALID_INCREMENT = 24H -POINT_STAT_SKIP_TIMES = -SKIP_TIMES = +POINT_STAT_SKIP_VALID_TIMES = +SKIP_VALID_TIMES = LEAD_SEQ = begin_end_incr({ENV[FHR_START]},{ENV[FHR_END]},{ENV[FHR_INCR]}) # Variables diff --git a/parm/metplus_config/stats/mesoscale/grid2obs/PointStat_fcstMESOSCALE_obsNDAS_PrepBufr.conf b/parm/metplus_config/stats/mesoscale/grid2obs/PointStat_fcstMESOSCALE_obsNDAS_PrepBufr.conf index 1fcad270a..5c4ba0438 100755 --- a/parm/metplus_config/stats/mesoscale/grid2obs/PointStat_fcstMESOSCALE_obsNDAS_PrepBufr.conf +++ b/parm/metplus_config/stats/mesoscale/grid2obs/PointStat_fcstMESOSCALE_obsNDAS_PrepBufr.conf @@ -20,8 +20,6 @@ OBS_POINT_STAT_INPUT_TEMPLATE = prepbufr.gfs.{valid?fmt=%Y%m%d%H}.nc POINT_STAT_OUTPUT_TEMPLATE = {MODEL}.{valid?fmt=%Y%m%d} [config] -LOG_LEVEL = {ENV[METPLUS_VERBOSITY]} -LOG_MET_VERBOSITY = {ENV[MET_VERBOSITY]} LOG_POINT_STAT_VERBOSITY = {LOG_MET_VERBOSITY} LOG_TIMESTAMP_TEMPLATE = %Y%m%d%H%M%S LOG_TIMESTAMP_USE_DATATIME = no @@ -46,8 +44,8 @@ VALID_TIME_FMT = %Y%m%d%H VALID_BEG = {ENV[VDATE]}{ENV[VHOUR]} VALID_END = {ENV[VDATE]}{ENV[VHOUR]} VALID_INCREMENT = 24H -POINT_STAT_SKIP_TIMES = -SKIP_TIMES = +POINT_STAT_SKIP_VALID_TIMES = +SKIP_VALID_TIMES = LEAD_SEQ = begin_end_incr({ENV[FHR_START]},{ENV[FHR_END]},{ENV[FHR_INCR]}) # Variables diff --git a/parm/metplus_config/stats/mesoscale/grid2obs/PointStat_fcstMESOSCALE_obsRAOB.conf b/parm/metplus_config/stats/mesoscale/grid2obs/PointStat_fcstMESOSCALE_obsRAOB.conf index 5bdf1bd19..a5582c1d4 100755 --- a/parm/metplus_config/stats/mesoscale/grid2obs/PointStat_fcstMESOSCALE_obsRAOB.conf +++ b/parm/metplus_config/stats/mesoscale/grid2obs/PointStat_fcstMESOSCALE_obsRAOB.conf @@ -25,8 +25,6 @@ POINT_STAT_OUTPUT_TEMPLATE = {MODEL}.{valid?fmt=%Y%m%d} POINT_STAT_VERIFICATION_MASK_TEMPLATE = {ENV[MASK_POLY_LIST]} [config] -LOG_LEVEL = {ENV[METPLUS_VERBOSITY]} -LOG_MET_VERBOSITY = {ENV[MET_VERBOSITY]} LOG_POINT_STAT_VERBOSITY = {LOG_MET_VERBOSITY} LOG_TIMESTAMP_TEMPLATE = %Y%m%d%H%M%S LOG_TIMESTAMP_USE_DATATIME = no @@ -50,8 +48,8 @@ VALID_TIME_FMT = %Y%m%d%H VALID_BEG = {ENV[VDATE]}{ENV[VHOUR]} VALID_END = {ENV[VDATE]}{ENV[VHOUR]} VALID_INCREMENT = 24H -POINT_STAT_SKIP_TIMES = -SKIP_TIMES = +POINT_STAT_SKIP_VALID_TIMES = +SKIP_VALID_TIMES = LEAD_SEQ = begin_end_incr({ENV[FHR_START]},{ENV[FHR_END]},{ENV[FHR_INCR]}) # Variables diff --git a/parm/metplus_config/stats/mesoscale/grid2obs/PointStat_fcstMESOSCALE_obsRAOB_VAR2.conf b/parm/metplus_config/stats/mesoscale/grid2obs/PointStat_fcstMESOSCALE_obsRAOB_VAR2.conf index cf4b1e220..21a5244e3 100755 --- a/parm/metplus_config/stats/mesoscale/grid2obs/PointStat_fcstMESOSCALE_obsRAOB_VAR2.conf +++ b/parm/metplus_config/stats/mesoscale/grid2obs/PointStat_fcstMESOSCALE_obsRAOB_VAR2.conf @@ -25,8 +25,6 @@ POINT_STAT_OUTPUT_TEMPLATE = {MODEL}.{valid?fmt=%Y%m%d} POINT_STAT_VERIFICATION_MASK_TEMPLATE = {ENV[MASK_POLY_LIST]} [config] -LOG_LEVEL = {ENV[METPLUS_VERBOSITY]} -LOG_MET_VERBOSITY = {ENV[MET_VERBOSITY]} LOG_POINT_STAT_VERBOSITY = {LOG_MET_VERBOSITY} LOG_TIMESTAMP_TEMPLATE = %Y%m%d%H%M%S LOG_TIMESTAMP_USE_DATATIME = no @@ -50,8 +48,8 @@ VALID_TIME_FMT = %Y%m%d%H VALID_BEG = {ENV[VDATE]}{ENV[VHOUR]} VALID_END = {ENV[VDATE]}{ENV[VHOUR]} VALID_INCREMENT = 24H -POINT_STAT_SKIP_TIMES = -SKIP_TIMES = +POINT_STAT_SKIP_VALID_TIMES = +SKIP_VALID_TIMES = LEAD_SEQ = begin_end_incr({ENV[FHR_START]},{ENV[FHR_END]},{ENV[FHR_INCR]}) # Variables diff --git a/parm/metplus_config/stats/mesoscale/grid2obs/RegridDataPlane_fcstMESOSCALE_PTYPE.conf b/parm/metplus_config/stats/mesoscale/grid2obs/RegridDataPlane_fcstMESOSCALE_PTYPE.conf index 2c1b0b767..862ccde6b 100644 --- a/parm/metplus_config/stats/mesoscale/grid2obs/RegridDataPlane_fcstMESOSCALE_PTYPE.conf +++ b/parm/metplus_config/stats/mesoscale/grid2obs/RegridDataPlane_fcstMESOSCALE_PTYPE.conf @@ -18,8 +18,6 @@ FCST_REGRID_DATA_PLANE_INPUT_TEMPLATE = {ENV[MODEL_INPUT_TEMPLATE]} FCST_REGRID_DATA_PLANE_OUTPUT_TEMPLATE = {MODEL}.{valid?fmt=%Y%m%d}/regrid_data_plane_{MODEL}_t{valid?fmt=%2H}z_{ENV[VERIF_TYPE]}_{ENV[NEST]}_{ENV[job_name]}_fhr{lead?fmt=%2H}.nc [config] -LOG_LEVEL = {ENV[METPLUS_VERBOSITY]} -LOG_MET_VERBOSITY = {ENV[MET_VERBOSITY]} LOG_REGRID_DATA_PLANE_VERBOSITY = {LOG_MET_VERBOSITY} LOG_TIMESTAMP_TEMPLATE = %Y%m%d%H%M%S LOG_TIMESTAMP_USE_DATATIME = no @@ -43,7 +41,7 @@ VALID_TIME_FMT = %Y%m%d%H VALID_BEG = {ENV[VDATE]}{ENV[VHOUR]} VALID_END = {ENV[VDATE]}{ENV[VHOUR]} VALID_INCREMENT = 24H -SKIP_TIMES = +SKIP_VALID_TIMES = LEAD_SEQ = begin_end_incr({ENV[FHR_START]},{ENV[FHR_END]},{ENV[FHR_INCR]}) FCST_REGRID_DATA_PLANE_RUN = True diff --git a/parm/metplus_config/stats/mesoscale/grid2obs/StatAnalysis_fcstMESOSCALE_GatherByCycle.conf b/parm/metplus_config/stats/mesoscale/grid2obs/StatAnalysis_fcstMESOSCALE_GatherByCycle.conf index 322c5fbf0..9d21bc2bc 100755 --- a/parm/metplus_config/stats/mesoscale/grid2obs/StatAnalysis_fcstMESOSCALE_GatherByCycle.conf +++ b/parm/metplus_config/stats/mesoscale/grid2obs/StatAnalysis_fcstMESOSCALE_GatherByCycle.conf @@ -13,8 +13,6 @@ STAT_ANALYSIS_OUTPUT_DIR = {OUTPUT_BASE} MODEL1_STAT_ANALYSIS_LOOKIN_DIR = {INPUT_BASE}/{MODEL1}.{ENV[VDATE]} [config] -LOG_LEVEL = {ENV[METPLUS_VERBOSITY]} -LOG_MET_VERBOSITY = {ENV[MET_VERBOSITY]} LOG_STAT_ANALYSIS_VERBOSITY = {LOG_MET_VERBOSITY} LOG_TIMESTAMP_TEMPLATE = %Y%m%d%H%M%S LOG_TIMESTAMP_USE_DATATIME = no @@ -43,8 +41,8 @@ VALID_END = {ENV[VDATE]} VALID_INCREMENT = 24H INIT_BEG = INIT_END = -STAT_ANALYSIS_SKIP_TIMES = -SKIP_TIMES = +STAT_ANALYSIS_SKIP_VALID_TIMES = +SKIP_VALID_TIMES = FCST_LEAD_LIST = FCST_VALID_HOUR_LIST = diff --git a/parm/metplus_config/stats/mesoscale/grid2obs/StatAnalysis_fcstMESOSCALE_GatherByDay.conf b/parm/metplus_config/stats/mesoscale/grid2obs/StatAnalysis_fcstMESOSCALE_GatherByDay.conf index 84ed12937..b2cfe668a 100644 --- a/parm/metplus_config/stats/mesoscale/grid2obs/StatAnalysis_fcstMESOSCALE_GatherByDay.conf +++ b/parm/metplus_config/stats/mesoscale/grid2obs/StatAnalysis_fcstMESOSCALE_GatherByDay.conf @@ -7,7 +7,7 @@ METPLUS_PARM_BASE = {METPLUS_BASE}/parm CONVERT = convert GEMPPAKTOCF_JAR = GempakToCF.jar -INPUT_BASE = {ENV[COMOUTsmall]} +INPUT_BASE = {ENV[COMOUTsmall]}/gather_small OUTPUT_BASE = {ENV[COMOUTfinal]} CONFIG_DIR = {ENV[MET_CONFIG]} LOG_DIR = {ENV[MET_PLUS_OUT]}/stat_analysis/logs @@ -18,8 +18,6 @@ STAT_ANALYSIS_OUTPUT_DIR = {OUTPUT_BASE} MODEL1_STAT_ANALYSIS_LOOKIN_DIR = {INPUT_BASE} [config] -LOG_LEVEL = {ENV[METPLUS_VERBOSITY]} -LOG_MET_VERBOSITY = {ENV[MET_VERBOSITY]} LOG_STAT_ANALYSIS_VERBOSITY = {LOG_MET_VERBOSITY} LOG_TIMESTAMP_TEMPLATE = %Y%m%d%H%M%S LOG_TIMESTAMP_USE_DATATIME = no @@ -48,8 +46,8 @@ VALID_END = {ENV[VDATE]} VALID_INCREMENT = 24H INIT_BEG = INIT_END = -STAT_ANALYSIS_SKIP_TIMES = -SKIP_TIMES = +STAT_ANALYSIS_SKIP_VALID_TIMES = +SKIP_VALID_TIMES = FCST_LEAD_LIST = FCST_VALID_HOUR_LIST = diff --git a/parm/metplus_config/stats/mesoscale/grid2obs/StatAnalysis_fcstMESOSCALE_obsMETAR_GatherByDay.conf b/parm/metplus_config/stats/mesoscale/grid2obs/StatAnalysis_fcstMESOSCALE_obsMETAR_GatherByDay.conf index 1440c25ed..e8c3fcb24 100755 --- a/parm/metplus_config/stats/mesoscale/grid2obs/StatAnalysis_fcstMESOSCALE_obsMETAR_GatherByDay.conf +++ b/parm/metplus_config/stats/mesoscale/grid2obs/StatAnalysis_fcstMESOSCALE_obsMETAR_GatherByDay.conf @@ -18,8 +18,6 @@ STAT_ANALYSIS_OUTPUT_DIR = {OUTPUT_BASE} MODEL1_STAT_ANALYSIS_LOOKIN_DIR = {INPUT_BASE}/{MODEL1}.{ENV[VDATE]} [config] -LOG_LEVEL = {ENV[METPLUS_VERBOSITY]} -LOG_MET_VERBOSITY = {ENV[MET_VERBOSITY]} LOG_STAT_ANALYSIS_VERBOSITY = {LOG_MET_VERBOSITY} LOG_TIMESTAMP_TEMPLATE = %Y%m%d%H%M%S LOG_TIMESTAMP_USE_DATATIME = no @@ -48,8 +46,8 @@ VALID_END = {ENV[VDATE]} VALID_INCREMENT = 24H INIT_BEG = INIT_END = -STAT_ANALYSIS_SKIP_TIMES = -SKIP_TIMES = +STAT_ANALYSIS_SKIP_VALID_TIMES = +SKIP_VALID_TIMES = FCST_LEAD_LIST = FCST_VALID_HOUR_LIST = diff --git a/parm/metplus_config/stats/mesoscale/grid2obs/StatAnalysis_fcstMESOSCALE_obsNDAS_GatherByDay.conf b/parm/metplus_config/stats/mesoscale/grid2obs/StatAnalysis_fcstMESOSCALE_obsNDAS_GatherByDay.conf index e50fa785d..78707fa92 100755 --- a/parm/metplus_config/stats/mesoscale/grid2obs/StatAnalysis_fcstMESOSCALE_obsNDAS_GatherByDay.conf +++ b/parm/metplus_config/stats/mesoscale/grid2obs/StatAnalysis_fcstMESOSCALE_obsNDAS_GatherByDay.conf @@ -17,8 +17,6 @@ LOG_METPLUS = {LOG_DIR}/{MODEL1}/metplus.statanalysis.log.{LOG_TIMESTAMP_TEMPLAT # times will loop through dates LOOP_ORDER = times PROCESS_LIST = StatAnalysis -LOG_MET_VERBOSITY = 4 -LOG_LEVEL=DEBUG LOOP_BY = VALID VALID_TIME_FMT = %Y%m%d diff --git a/parm/metplus_config/stats/mesoscale/grid2obs/StatAnalysis_fcstMESOSCALE_obsRAOB_GatherByDay.conf b/parm/metplus_config/stats/mesoscale/grid2obs/StatAnalysis_fcstMESOSCALE_obsRAOB_GatherByDay.conf index 27ab3080d..36d6a63fb 100644 --- a/parm/metplus_config/stats/mesoscale/grid2obs/StatAnalysis_fcstMESOSCALE_obsRAOB_GatherByDay.conf +++ b/parm/metplus_config/stats/mesoscale/grid2obs/StatAnalysis_fcstMESOSCALE_obsRAOB_GatherByDay.conf @@ -18,8 +18,6 @@ STAT_ANALYSIS_OUTPUT_DIR = {OUTPUT_BASE} MODEL1_STAT_ANALYSIS_LOOKIN_DIR = {INPUT_BASE}/{MODEL1}.{ENV[VDATE]} [config] -LOG_LEVEL = {ENV[METPLUS_VERBOSITY]} -LOG_MET_VERBOSITY = {ENV[MET_VERBOSITY]} LOG_STAT_ANALYSIS_VERBOSITY = {LOG_MET_VERBOSITY} LOG_TIMESTAMP_TEMPLATE = %Y%m%d%H%M%S LOG_TIMESTAMP_USE_DATATIME = no @@ -48,8 +46,8 @@ VALID_END = {ENV[VDATE]} VALID_INCREMENT = 24H INIT_BEG = INIT_END = -STAT_ANALYSIS_SKIP_TIMES = -SKIP_TIMES = +STAT_ANALYSIS_SKIP_VALID_TIMES = +SKIP_VALID_TIMES = FCST_LEAD_LIST = FCST_VALID_HOUR_LIST = diff --git a/scripts/plots/mesoscale/exevs_mesoscale_grid2obs_plots.sh b/scripts/plots/mesoscale/exevs_mesoscale_grid2obs_plots.sh index af69e7fb3..af728225d 100755 --- a/scripts/plots/mesoscale/exevs_mesoscale_grid2obs_plots.sh +++ b/scripts/plots/mesoscale/exevs_mesoscale_grid2obs_plots.sh @@ -61,7 +61,7 @@ if [ $USE_CFP = YES ]; then if [ $machine = WCOSS2 ]; then nselect=$(cat $PBS_NODEFILE | wc -l) nnp=$(($nselect * $nproc)) - launcher="mpiexec -np ${nnp} -ppn ${nproc} --cpu-bind verbose,depth cfp" + launcher="mpiexec -np ${nnp} -ppn ${nproc} -depth 2 --cpu-bind verbose,depth cfp" # launcher="mpiexec -np $nproc -ppn $nproc --cpu-bind verbose,depth cfp" # ---- elif [$machine = HERA -o $machine = ORION -o $machine = S4 -o $machine = JET ]; then diff --git a/scripts/stats/mesoscale/exevs_mesoscale_nam_grid2obs_stats.sh b/scripts/stats/mesoscale/exevs_mesoscale_nam_grid2obs_stats.sh index 135f77b25..1d6008652 100755 --- a/scripts/stats/mesoscale/exevs_mesoscale_nam_grid2obs_stats.sh +++ b/scripts/stats/mesoscale/exevs_mesoscale_nam_grid2obs_stats.sh @@ -73,6 +73,10 @@ echo "*****************************" # Create Output Directories python $USHevs/mesoscale/mesoscale_create_output_dirs.py export err=$?; err_chk + + # Preprocess Prepbufr Data + python $USHevs/mesoscale/mesoscale_stats_grid2obs_preprocess_prepbufr.py + export err=$?; err_chk # Create Reformat Job Script python $USHevs/mesoscale/mesoscale_stats_grid2obs_create_job_script.py @@ -220,10 +224,6 @@ echo "*****************************" export evs_run_mode=$evs_run_mode source $config - if [ ${#VAR_NAME_LIST} -lt 1 ]; then - continue - fi - # Create Output Directories python $USHevs/mesoscale/mesoscale_create_output_dirs.py export err=$?; err_chk @@ -280,20 +280,16 @@ echo "*****************************" echo "Gather jobs done" echo "*****************************" -# Copy stat output files to EVS COMOUTsmall directory - if [ $SENDCOM = YES ]; then - for VERIF_TYPE in $VERIF_TYPES;do - for MODEL_DIR_PATH in $MET_PLUS_OUT/$VERIF_TYPE/point_stat/$MODELNAME*; do - if [ -d $MODEL_DIR_PATH ]; then - MODEL_DIR=$(echo ${MODEL_DIR_PATH##*/}) - mkdir -p $COMOUTsmall - for FILE in $MODEL_DIR_PATH/*; do - cp -v $FILE $COMOUTsmall/. - done - fi - done - done - fi +# Copy "gather" output files to EVS COMOUTsmall directory +if [ $SENDCOM = YES ]; then + for MODEL_DIR_PATH in $MET_PLUS_OUT/gather_small/stat_analysis/$MODELNAME*; do + for FILE in $MODEL_DIR_PATH/*; do + if [ -s "$FILE" ]; then + cp -v $FILE $COMOUTsmall/gather_small/. + fi + done + done +fi echo "*****************************" echo "Gather3 jobs begin" @@ -356,21 +352,16 @@ echo "*****************************" echo "Gather3 jobs done" echo "*****************************" -# Copy output files into the correct EVS COMOUT directory +# Copy "gather" output files to EVS COMOUTsmall directory if [ $SENDCOM = YES ]; then for MODEL_DIR_PATH in $MET_PLUS_OUT/gather_small/stat_analysis/$MODELNAME*; do - MODEL_DIR=$(echo ${MODEL_DIR_PATH##*/}) - mkdir -p $COMOUT/$MODEL_DIR - for FILE in $MODEL_DIR_PATH/*; do - if [ -s $FILE ]; then - cp -v $FILE $COMOUT/$MODEL_DIR/. - fi - done + for FILE in $MODEL_DIR_PATH/*; do + if [ -s "$FILE" ]; then + cp -v $FILE $COMOUTsmall/gather_small/. + fi + done done - fi + fi echo "******************************" -echo "Begin to print METplus Log files " - cat $DATA/grid2obs/METplus_output/*/*/pb2nc/logs/* -echo "End to print METplus Log files " diff --git a/scripts/stats/mesoscale/exevs_mesoscale_nam_precip_stats.sh b/scripts/stats/mesoscale/exevs_mesoscale_nam_precip_stats.sh index 629897f07..4e55dd85b 100755 --- a/scripts/stats/mesoscale/exevs_mesoscale_nam_precip_stats.sh +++ b/scripts/stats/mesoscale/exevs_mesoscale_nam_precip_stats.sh @@ -90,9 +90,4 @@ for group in $JOB_GROUP_list; do fi done -echo "******************************" -echo "Begin to print METplus Log files " - cat $DATA/logs/* -echo "End to print METplus Log files " - diff --git a/scripts/stats/mesoscale/exevs_mesoscale_nam_snowfall_stats.sh b/scripts/stats/mesoscale/exevs_mesoscale_nam_snowfall_stats.sh index a7667baae..e2d932df4 100755 --- a/scripts/stats/mesoscale/exevs_mesoscale_nam_snowfall_stats.sh +++ b/scripts/stats/mesoscale/exevs_mesoscale_nam_snowfall_stats.sh @@ -90,8 +90,4 @@ for group in $JOB_GROUP_list; do fi done -echo "******************************" -echo "Begin to print METplus Log files " - cat $DATA/logs/* -echo "End to print METplus Log files " diff --git a/scripts/stats/mesoscale/exevs_mesoscale_rap_grid2obs_stats.sh b/scripts/stats/mesoscale/exevs_mesoscale_rap_grid2obs_stats.sh index e0b46fdf0..6dca65ad2 100755 --- a/scripts/stats/mesoscale/exevs_mesoscale_rap_grid2obs_stats.sh +++ b/scripts/stats/mesoscale/exevs_mesoscale_rap_grid2obs_stats.sh @@ -24,7 +24,7 @@ set -x # Set env export OBSDIR=OBS - export fcstmax=48 + export fcstmax=51 export model1=`echo $MODELNAME | tr a-z A-Z` export model0=`echo $MODELNAME | tr A-Z a-z` @@ -58,7 +58,7 @@ echo "*****************************" # Check for restart files reformat echo " Check for restart files reformat begin" if [ $evs_run_mode = production ]; then - ${USHevs}/mesoscale/mesoscale_stats_g2o_production_restart.sh + ${USHevs}/mesoscale/mesoscale_stats_g2o_production_restart.sh export err=$?; err_chk fi echo " Check for restart files reformat done" @@ -76,6 +76,10 @@ echo "*****************************" # Create Output Directories python $USHevs/mesoscale/mesoscale_create_output_dirs.py export err=$?; err_chk + + # Preprocess Prepbufr Data + python $USHevs/mesoscale/mesoscale_stats_grid2obs_preprocess_prepbufr.py + export err=$?; err_chk # Create Reformat Job Script python $USHevs/mesoscale/mesoscale_stats_grid2obs_create_job_script.py @@ -283,20 +287,17 @@ echo "*****************************" echo "Gather jobs done" echo "*****************************" -# Copy stat output files to EVS COMOUTsmall directory - if [ $SENDCOM = YES ]; then - for VERIF_TYPE in $VERIF_TYPES;do - for MODEL_DIR_PATH in $MET_PLUS_OUT/$VERIF_TYPE/point_stat/$MODELNAME*; do - if [ -d $MODEL_DIR_PATH ]; then - MODEL_DIR=$(echo ${MODEL_DIR_PATH##*/}) - mkdir -p $COMOUTsmall - for FILE in $MODEL_DIR_PATH/*; do - cp -v $FILE $COMOUTsmall/. - done +# Copy "gather" output files to EVS COMOUTsmall directory +if [ $SENDCOM = YES ]; then + for MODEL_DIR_PATH in $MET_PLUS_OUT/gather_small/stat_analysis/$MODELNAME*; do + for FILE in $MODEL_DIR_PATH/*; do + if [ -s "$FILE" ]; then + cp -v $FILE $COMOUTsmall/gather_small/. fi - done - done - fi + done + done +fi + echo "*****************************" echo "Gather3 jobs begin" @@ -311,7 +312,7 @@ echo "*****************************" # Create Output Directories python $USHevs/mesoscale/mesoscale_create_output_dirs.py export err=$?; err_chk - + # Create Gather 3 Job Script python $USHevs/mesoscale/mesoscale_stats_grid2obs_create_job_script.py export err=$?; err_chk @@ -359,21 +360,17 @@ echo "*****************************" echo "Gather3 jobs done" echo "*****************************" -# Copy output files into the correct EVS COMOUT directory - if [ $SENDCOM = YES ]; then - for MODEL_DIR_PATH in $MET_PLUS_OUT/gather_small/stat_analysis/$MODELNAME*; do - MODEL_DIR=$(echo ${MODEL_DIR_PATH##*/}) - mkdir -p $COMOUT/$MODEL_DIR - for FILE in $MODEL_DIR_PATH/*; do - if [ -s $FILE ]; then - cp -v $FILE $COMOUT/$MODEL_DIR/. - fi - done - done - fi +# Copy "gather" output files to EVS COMOUTsmall directory +if [ $SENDCOM = YES ]; then + for MODEL_DIR_PATH in $MET_PLUS_OUT/gather_small/stat_analysis/$MODELNAME*; do + for FILE in $MODEL_DIR_PATH/*; do + if [ -s "$FILE" ]; then + cp -v $FILE $COMOUTsmall/gather_small/. + fi + done + done +fi + echo "******************************" -echo "Begin to print METplus Log files " - cat $DATA/grid2obs/METplus_output/*/*/pb2nc/logs/* -echo "End to print METplus Log files " diff --git a/scripts/stats/mesoscale/exevs_mesoscale_rap_precip_stats.sh b/scripts/stats/mesoscale/exevs_mesoscale_rap_precip_stats.sh index 91458b70e..daedab9ab 100755 --- a/scripts/stats/mesoscale/exevs_mesoscale_rap_precip_stats.sh +++ b/scripts/stats/mesoscale/exevs_mesoscale_rap_precip_stats.sh @@ -90,9 +90,5 @@ for group in $JOB_GROUP_list; do fi done -echo "******************************" -echo "Begin to print METplus Log files " - cat $DATA/logs/* -echo "End to print METplus Log files " diff --git a/scripts/stats/mesoscale/exevs_mesoscale_rap_snowfall_stats.sh b/scripts/stats/mesoscale/exevs_mesoscale_rap_snowfall_stats.sh index 1b4af5826..70574d4e7 100755 --- a/scripts/stats/mesoscale/exevs_mesoscale_rap_snowfall_stats.sh +++ b/scripts/stats/mesoscale/exevs_mesoscale_rap_snowfall_stats.sh @@ -90,8 +90,4 @@ for group in $JOB_GROUP_list; do fi done -echo "******************************" -echo "Begin to print METplus Log files " - cat $DATA/logs/* -echo "End to print METplus Log files " diff --git a/ush/mesoscale/mesoscale_check_input_data.py b/ush/mesoscale/mesoscale_check_input_data.py index 7288f88d6..b5d80e466 100644 --- a/ush/mesoscale/mesoscale_check_input_data.py +++ b/ush/mesoscale/mesoscale_check_input_data.py @@ -21,22 +21,6 @@ print(f"BEGIN: {os.path.basename(__file__)}") -# Construct a file name given a template -def fname_constructor(template_str, IDATE="YYYYmmdd", IHOUR="HH", - VDATE="YYYYmmdd", VHOUR="HH", VDATEHOUR="YYYYmmddHH", - VDATEm1H="YYYYmmdd", VDATEHOURm1H="YYYYmmddHH", - FHR="HH", LVL="0"): - template_str = template_str.replace('{IDATE}', IDATE) - template_str = template_str.replace('{IHOUR}', IHOUR) - template_str = template_str.replace('{VDATE}', VDATE) - template_str = template_str.replace('{VHOUR}', VHOUR) - template_str = template_str.replace('{VDATEHOUR}', VDATEHOUR) - template_str = template_str.replace('{VDATEm1H}', VDATEm1H) - template_str = template_str.replace('{VDATEHOURm1H}', VDATEHOURm1H) - template_str = template_str.replace('{FHR}', FHR) - template_str = template_str.replace('{LVL}', LVL) - return template_str - # Determine whether or not to proceed STEP = os.environ['STEP'] @@ -289,7 +273,7 @@ def fname_constructor(template_str, IDATE="YYYYmmdd", IHOUR="HH", IDATE = inits_list[v][i].strftime('%Y%m%d') IHOUR = inits_list[v][i].strftime('%H') for template in fcst_templates: - fcst_paths.append(fname_constructor( + fcst_paths.append(cutil.fname_constructor( template, IDATE=IDATE, IHOUR=IHOUR, FHR=str(int(FHR)).zfill(2) )) @@ -422,30 +406,10 @@ def fname_constructor(template_str, IDATE="YYYYmmdd", IHOUR="HH", # Check for missing obs data (analyses or da) # Make list of paths - anl_templates = [] + anl_paths= [] if STEP == 'stats': if VERIF_CASE == 'grid2obs': - # Expect PrepBufr to at least be available at these times - if VHOUR in ['00', '06', '12', '18']: - anl_templates.append(os.path.join( - COMINobsproc, - 'nam.{VDATE}', - 'nam.t{VHOUR}z.prepbufr.tm00' - )) - anl_templates.append(os.path.join( - COMINobsproc, - 'gdas.{VDATE}', - '{VHOUR}', - 'atmos', - 'gdas.t{VHOUR}z.prepbufr' - )) - anl_paths = [] - for v, vdate in enumerate(vdates): - for template in anl_templates: - anl_paths.append(fname_constructor( - template, VDATE=vdate.strftime('%Y%m%d'), VHOUR=vdate.strftime('%H') - )) - anl_paths = np.unique(anl_paths) + anl_paths = cutil.get_prepbufr_templates(COMINobsproc, vdates, paths=anl_paths) # Record paths that don't exist missing_anl_paths = [] diff --git a/ush/mesoscale/mesoscale_check_settings.py b/ush/mesoscale/mesoscale_check_settings.py index 361c80e16..11e3c14c4 100644 --- a/ush/mesoscale/mesoscale_check_settings.py +++ b/ush/mesoscale/mesoscale_check_settings.py @@ -55,10 +55,10 @@ evs_mesoscale_settings_dict['shared'] = [] evs_mesoscale_settings_dict['modules'] = ['MET_PLUS_PATH', 'MET_PATH', 'MET_CONFIG'] evs_mesoscale_settings_dict['RUN_GRID2OBS_PREP'] = [ - 'MET_PLUS_CONF','MET_PLUS_OUT','METPLUS_VERBOSITY','MET_VERBOSITY', + 'MET_PLUS_CONF','MET_PLUS_OUT', 'LOG_MET_OUTPUT_TO_METPLUS','NEST','TEMP_DIR','GRID_DIR','URL_HEAD', ] -evs_mesoscale_settings_dict['RUN_GRID2OBS_STATS'] = [] +evs_mesoscale_settings_dict['RUN_GRID2OBS_STATS'] = ['bufr_ROOT'] evs_mesoscale_settings_dict['RUN_GRID2OBS_PLOTS'] = [ 'MET_VERSION','IMG_HEADER','PRUNE_DIR','SAVE_DIR','LOG_TEMPLATE', 'LOG_LEVEL','STAT_OUTPUT_BASE_DIR','STAT_OUTPUT_BASE_TEMPLATE', @@ -69,7 +69,7 @@ ] evs_mesoscale_settings_dict['RUN_PRECIP_STATS'] = [ 'MET_PLUS_CONF','MET_PLUS_OUT','MET_CONFIG_OVERRIDES', - 'METPLUS_VERBOSITY','MET_VERBOSITY','LOG_MET_OUTPUT_TO_METPLUS', + 'LOG_MET_OUTPUT_TO_METPLUS', 'VHOUR','FHR_END_SHORT','FHR_INCR_SHORT','FHR_END_FULL', 'FHR_INCR_FULL','COMINfcst','COMINobs','NEST', 'OBSNAME','MIN_IHOUR','MODEL_ACC','OBS_ACC','ACC','BOOL_NBRHD','FCST_LEV', @@ -138,21 +138,9 @@ } if STEP.upper() == 'PREP': if VERIF_CASE.upper() == 'GRID2OBS': - valid_config_var_values_dict['METPLUS_VERBOSITY'] = [ - 'DEBUG', 'INFO', 'WARN', 'ERROR' - ] - valid_config_var_values_dict['MET_VERBOSITY'] = [ - '0', '1', '2', '3', '4', '5' - ] valid_config_var_values_dict['LOG_MET_OUTPUT_TO_METPLUS'] = ['yes', 'no'] valid_config_var_values_dict['NEST'] = ['spc_otlk'] if STEP.upper() == 'STATS': - valid_config_var_values_dict['METPLUS_VERBOSITY'] = [ - 'DEBUG', 'INFO', 'WARN', 'ERROR' - ] - valid_config_var_values_dict['MET_VERBOSITY'] = [ - '0', '1', '2', '3', '4', '5' - ] valid_config_var_values_dict['LOG_MET_OUTPUT_TO_METPLUS'] = ['yes', 'no'] valid_config_var_values_dict['NEST'] = ['ak', 'conus', 'subreg', 'spc_otlk', 'firewx', 'hi', 'pr', 'gu','namer','conusc','akc','conusp'] valid_config_var_values_dict['BOOL_NBRHD'] = ['True', 'False'] diff --git a/ush/mesoscale/mesoscale_create_output_dirs.py b/ush/mesoscale/mesoscale_create_output_dirs.py index 6f2170c24..e85914fc8 100644 --- a/ush/mesoscale/mesoscale_create_output_dirs.py +++ b/ush/mesoscale/mesoscale_create_output_dirs.py @@ -52,6 +52,7 @@ start_date_dt = vdate_dt - td(hours=fhr_end_max) VERIF_TYPE = os.environ['VERIF_TYPE'] OBSNAME = os.environ['OBSNAME'] + COMOUTsmall = os.environ['COMOUTsmall'] elif STEP == 'plots': all_eval_periods = cutil.get_all_eval_periods(graphics_pcp) COMOUTplots = os.environ['COMOUTplots'] @@ -66,6 +67,7 @@ start_date_dt = vdate_dt - td(hours=fhr_end_max) VERIF_TYPE = os.environ['VERIF_TYPE'] OBSNAME = os.environ['OBSNAME'] + COMOUTsmall = os.environ['COMOUTsmall'] elif STEP == 'plots': all_eval_periods = cutil.get_all_eval_periods(graphics_g2o) COMOUTplots = os.environ['COMOUTplots'] @@ -102,6 +104,9 @@ data_dir_list.append(os.path.join(data_base_dir, MODELNAME)) data_dir_list.append(os.path.join(data_base_dir, MODELNAME, 'merged_ptype')) data_dir_list.append(os.path.join(data_base_dir, MODELNAME, 'tmp')) + data_dir_list.append(os.path.join( + data_base_dir, OBSNAME, 'prepbufr' + )) elif VERIF_CASE == 'snowfall': if STEP == 'stats': pass @@ -207,11 +212,9 @@ working_output_base_dir, 'stat_analysis', MODELNAME+'.'+vdate_dt.strftime('%Y%m%d') )) - date_dt = start_date_dt - while date_dt <= vdate_dt+td(days=1): COMOUT_dir_list.append(os.path.join( COMOUT, - MODELNAME+'.'+date_dt.strftime('%Y%m%d') + MODELNAME+'.'+vdate_dt.strftime('%Y%m%d') )) if job_type == 'reformat': working_dir_list.append(os.path.join( @@ -222,7 +225,6 @@ working_output_base_dir, 'pcp_combine', MODELNAME+'.'+date_dt.strftime('init%Y%m%d') )) - date_dt+=td(days=1) elif VERIF_CASE == "grid2obs": if job_type == 'reformat': working_output_base_dir = os.path.join( @@ -310,22 +312,23 @@ working_output_base_dir, 'stat_analysis', MODELNAME+'.'+vdate_dt.strftime('%Y%m%d') )) - date_dt = start_date_dt - while date_dt <= vdate_dt+td(days=1): + COMOUT_dir_list.append(os.path.join( + COMOUTsmall, + 'gather_small' + )) COMOUT_dir_list.append(os.path.join( COMOUT, - MODELNAME+'.'+date_dt.strftime('%Y%m%d') + MODELNAME+'.'+vdate_dt.strftime('%Y%m%d') )) if job_type == 'reformat': working_dir_list.append(os.path.join( working_output_base_dir, NEST, 'pb2nc', - OBSNAME+'.'+date_dt.strftime('%Y%m%d') + OBSNAME+'.'+vdate_dt.strftime('%Y%m%d') )) working_dir_list.append(os.path.join( working_output_base_dir, NEST, 'pb2nc', - MODELNAME+'.'+date_dt.strftime('init%Y%m%d') + MODELNAME+'.'+vdate_dt.strftime('init%Y%m%d') )) - date_dt+=td(days=1) elif VERIF_CASE == "snowfall": pass elif STEP == 'plots': diff --git a/ush/mesoscale/mesoscale_stats_grid2obs_create_job_script.py b/ush/mesoscale/mesoscale_stats_grid2obs_create_job_script.py index 2603896b1..e50344bc2 100644 --- a/ush/mesoscale/mesoscale_stats_grid2obs_create_job_script.py +++ b/ush/mesoscale/mesoscale_stats_grid2obs_create_job_script.py @@ -40,8 +40,6 @@ MET_PLUS_CONF = os.environ['MET_PLUS_CONF'] MET_PLUS_OUT = os.environ['MET_PLUS_OUT'] MET_CONFIG_OVERRIDES = os.environ['MET_CONFIG_OVERRIDES'] -METPLUS_VERBOSITY = os.environ['METPLUS_VERBOSITY'] -MET_VERBOSITY = os.environ['MET_VERBOSITY'] metplus_launcher = 'run_metplus.py' machine_conf = os.path.join( os.environ['PARMevs'], 'metplus_config', 'machine.conf' @@ -172,8 +170,6 @@ 'MET_PLUS_CONF': MET_PLUS_CONF, 'MET_PLUS_OUT': MET_PLUS_OUT, 'MET_CONFIG_OVERRIDES': MET_CONFIG_OVERRIDES, - 'METPLUS_VERBOSITY': METPLUS_VERBOSITY, - 'MET_VERBOSITY': MET_VERBOSITY, } job_iterate_over_env_lists_dict = {} job_iterate_over_custom_lists_dict = {} diff --git a/ush/mesoscale/mesoscale_stats_grid2obs_preprocess_prepbufr.py b/ush/mesoscale/mesoscale_stats_grid2obs_preprocess_prepbufr.py new file mode 100644 index 000000000..f7b3723ec --- /dev/null +++ b/ush/mesoscale/mesoscale_stats_grid2obs_preprocess_prepbufr.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +''' +Name: mesoscale_stats_grid2obs_preprocess_prepbufr.py +Contact(s): Marcel Caron (marcel.caron@noaa.gov) +Abstract: Preprocess input prepbufr files and store the preprocessed file in + DATA. Preprocessing currently includes splitting each file into + multiple files by subset (i.e. message type, e.g., ADPUPA), and + concatenating the desired subsets to a new smaller file. +Run By: scripts/stats/mesoscale/exevs_nam_grid2obs_stats.sh + scripts/stats/mesoscale/exevs_rap_grid2obs_stats.sh +''' +import os +from datetime import datetime +import mesoscale_util as cutil + +print("BEGIN: "+os.path.basename(__file__)) + +# Run split_by_subset on all INPUT_FILES, saving SUBSETS we want to OUTPUT_DIT +DATA = os.environ['DATA'] +COMINobsproc = os.environ['COMINobsproc'] +MODELNAME = os.environ['MODELNAME'] +VERIF_CASE = os.environ['VERIF_CASE'] +VERIF_TYPE = os.environ['VERIF_TYPE'] +VDATE = os.environ['VDATE'] +VHOUR = os.environ['VHOUR'] + +workdir = os.path.join(DATA, VERIF_CASE, 'data', MODELNAME, 'tmp') +outdir = os.path.join(DATA, VERIF_CASE, 'data', VERIF_TYPE, 'prepbufr') +if VERIF_TYPE == 'raob': + subsets = ['ADPUPA'] +elif VERIF_TYPE == 'metar': + subsets = ['ADPSFC'] +else: + raise ValueError( + f'\"{VERIF_TYPE}\" is not a valid VERIF_TYPE for ' + + '{os.path.basename(__file__)}' + ) +vdate = datetime.strptime(VDATE+VHOUR, '%Y%m%d%H') +infiles = cutil.get_prepbufr_templates(COMINobsproc, [vdate]) + +for infile in infiles: + indir, fname = os.path.split(infile) + cutil.preprocess_prepbufr( + indir, fname, workdir, outdir, subsets + ) + +print("END: "+os.path.basename(__file__)) diff --git a/ush/mesoscale/mesoscale_stats_grid2obs_var_defs.py b/ush/mesoscale/mesoscale_stats_grid2obs_var_defs.py index 44e812249..dd59afae6 100644 --- a/ush/mesoscale/mesoscale_stats_grid2obs_var_defs.py +++ b/ush/mesoscale/mesoscale_stats_grid2obs_var_defs.py @@ -4,45 +4,29 @@ 'raob': { 'nam': { 'var1_fcst_name': 'HGT', - 'var1_fcst_levels': ("'P1000, P975, P950, P925, P900, P875, " - + "P850, P825, P800, P775, P750, P725, " - + "P700, P675, P650, P625, P600, P575, " - + "P550, P525, P500, P475, P450, P425, " - + "P400, P375, P350, P325, P300, P275, " - + "P250, P225, P200, P175, P150, P125, " - + "P100, P75, P50, P30, P20, P10'"), + 'var1_fcst_levels': ("'P1000, P925, P850, P700, P500, P400, " + + "P300, P250, P200, P150, P100, P50, " + + "P20, P10'"), 'var1_fcst_thresholds': '', 'var1_fcst_options': '', 'var1_obs_name': 'HGT', - 'var1_obs_levels': ("'P1000, P975, P950, P925, P900, P875, " - + "P850, P825, P800, P775, P750, P725, " - + "P700, P675, P650, P625, P600, P575, " - + "P550, P525, P500, P475, P450, P425, " - + "P400, P375, P350, P325, P300, P275, " - + "P250, P225, P200, P175, P150, P125, " - + "P100, P75, P50, P30, P20, P10'"), + 'var1_obs_levels': ("'P1000, P925, P850, P700, P500, P400, " + + "P300, P250, P200, P150, P100, P50, " + + "P20, P10'"), 'var1_obs_thresholds': '', 'var1_obs_options': '', }, 'rap': { 'var1_fcst_name': 'HGT', - 'var1_fcst_levels': ("'P1000, P975, P950, P925, P900, P875, " - + "P850, P825, P800, P775, P750, P725, " - + "P700, P675, P650, P625, P600, P575, " - + "P550, P525, P500, P475, P450, P425, " - + "P400, P375, P350, P325, P300, P275, " - + "P250, P225, P200, P175, P150, P125, " - + "P100, P75, P50, P30, P20, P10'"), + 'var1_fcst_levels': ("'P1000, P925, P850, P700, P500, P400, " + + "P300, P250, P200, P150, P100, " + + "P50'"), 'var1_fcst_thresholds': '', 'var1_fcst_options': '', 'var1_obs_name': 'HGT', - 'var1_obs_levels': ("'P1000, P975, P950, P925, P900, P875, " - + "P850, P825, P800, P775, P750, P725, " - + "P700, P675, P650, P625, P600, P575, " - + "P550, P525, P500, P475, P450, P425, " - + "P400, P375, P350, P325, P300, P275, " - + "P250, P225, P200, P175, P150, P125, " - + "P100, P75, P50, P30, P20, P10'"), + 'var1_obs_levels': ("'P1000, P925, P850, P700, P500, P400, " + + "P300, P250, P200, P150, P100, " + + "P50'"), 'var1_obs_thresholds': '', 'var1_obs_options': '', }, @@ -61,45 +45,29 @@ 'raob': { 'nam': { 'var1_fcst_name': 'TMP', - 'var1_fcst_levels': ("'P1000, P975, P950, P925, P900, P875, " - + "P850, P825, P800, P775, P750, P725, " - + "P700, P675, P650, P625, P600, P575, " - + "P550, P525, P500, P475, P450, P425, " - + "P400, P375, P350, P325, P300, P275, " - + "P250, P225, P200, P175, P150, P125, " - + "P100, P75, P50, P30, P20, P10'"), + 'var1_fcst_levels': ("'P1000, P925, P850, P700, P500, P400, " + + "P300, P250, P200, P150, P100, P50, " + + "P20, P10'"), 'var1_fcst_thresholds': '', 'var1_fcst_options': '', 'var1_obs_name': 'TMP', - 'var1_obs_levels': ("'P1000, P975, P950, P925, P900, P875, " - + "P850, P825, P800, P775, P750, P725, " - + "P700, P675, P650, P625, P600, P575, " - + "P550, P525, P500, P475, P450, P425, " - + "P400, P375, P350, P325, P300, P275, " - + "P250, P225, P200, P175, P150, P125, " - + "P100, P75, P50, P30, P20, P10'"), + 'var1_obs_levels': ("'P1000, P925, P850, P700, P500, P400, " + + "P300, P250, P200, P150, P100, P50, " + + "P20, P10'"), 'var1_obs_thresholds': '', 'var1_obs_options': '', }, 'rap': { 'var1_fcst_name': 'TMP', - 'var1_fcst_levels': ("'P1000, P975, P950, P925, P900, P875, " - + "P850, P825, P800, P775, P750, P725, " - + "P700, P675, P650, P625, P600, P575, " - + "P550, P525, P500, P475, P450, P425, " - + "P400, P375, P350, P325, P300, P275, " - + "P250, P225, P200, P175, P150, P125, " - + "P100, P75, P50, P30, P20, P10'"), + 'var1_fcst_levels': ("'P1000, P925, P850, P700, P500, P400, " + + "P300, P250, P200, P150, P100, " + + "P50'"), 'var1_fcst_thresholds': '', 'var1_fcst_options': '', 'var1_obs_name': 'TMP', - 'var1_obs_levels': ("'P1000, P975, P950, P925, P900, P875, " - + "P850, P825, P800, P775, P750, P725, " - + "P700, P675, P650, P625, P600, P575, " - + "P550, P525, P500, P475, P450, P425, " - + "P400, P375, P350, P325, P300, P275, " - + "P250, P225, P200, P175, P150, P125, " - + "P100, P75, P50, P30, P20, P10'"), + 'var1_obs_levels': ("'P1000, P925, P850, P700, P500, P400, " + + "P300, P250, P200, P150, P100, " + + "P50'"), 'var1_obs_thresholds': '', 'var1_obs_options': '', }, @@ -118,85 +86,53 @@ 'raob': { 'nam': { 'var1_fcst_name': 'UGRD', - 'var1_fcst_levels': ("'P1000, P975, P950, P925, P900, P875, " - + "P850, P825, P800, P775, P750, P725, " - + "P700, P675, P650, P625, P600, P575, " - + "P550, P525, P500, P475, P450, P425, " - + "P400, P375, P350, P325, P300, P275, " - + "P250, P225, P200, P175, P150, P125, " - + "P100, P75, P50, P30, P20, P10'"), + 'var1_fcst_levels': ("'P1000, P925, P850, P700, P500, P400, " + + "P300, P250, P200, P150, P100, P50, " + + "P20, P10'"), 'var1_fcst_thresholds': '', 'var1_fcst_options': '', 'var1_obs_name': 'UGRD', - 'var1_obs_levels': ("'P1000, P975, P950, P925, P900, P875, " - + "P850, P825, P800, P775, P750, P725, " - + "P700, P675, P650, P625, P600, P575, " - + "P550, P525, P500, P475, P450, P425, " - + "P400, P375, P350, P325, P300, P275, " - + "P250, P225, P200, P175, P150, P125, " - + "P100, P75, P50, P30, P20, P10'"), + 'var1_obs_levels': ("'P1000, P925, P850, P700, P500, P400, " + + "P300, P250, P200, P150, P100, P50, " + + "P20, P10'"), 'var1_obs_thresholds': '', 'var1_obs_options': '', 'var2_fcst_name': 'VGRD', - 'var2_fcst_levels': ("'P1000, P975, P950, P925, P900, P875, " - + "P850, P825, P800, P775, P750, P725, " - + "P700, P675, P650, P625, P600, P575, " - + "P550, P525, P500, P475, P450, P425, " - + "P400, P375, P350, P325, P300, P275, " - + "P250, P225, P200, P175, P150, P125, " - + "P100, P75, P50, P30, P20, P10'"), + 'var2_fcst_levels': ("'P1000, P925, P850, P700, P500, P400, " + + "P300, P250, P200, P150, P100, P50, " + + "P20, P10'"), 'var2_fcst_thresholds': '', 'var2_fcst_options': '', 'var2_obs_name': 'VGRD', - 'var2_obs_levels': ("'P1000, P975, P950, P925, P900, P875, " - + "P850, P825, P800, P775, P750, P725, " - + "P700, P675, P650, P625, P600, P575, " - + "P550, P525, P500, P475, P450, P425, " - + "P400, P375, P350, P325, P300, P275, " - + "P250, P225, P200, P175, P150, P125, " - + "P100, P75, P50, P30, P20, P10'"), + 'var2_obs_levels': ("'P1000, P925, P850, P700, P500, P400, " + + "P300, P250, P200, P150, P100, P50, " + + "P20, P10'"), 'var2_obs_thresholds': '', 'var2_obs_options': '', }, 'rap': { 'var1_fcst_name': 'UGRD', - 'var1_fcst_levels': ("'P1000, P975, P950, P925, P900, P875, " - + "P850, P825, P800, P775, P750, P725, " - + "P700, P675, P650, P625, P600, P575, " - + "P550, P525, P500, P475, P450, P425, " - + "P400, P375, P350, P325, P300, P275, " - + "P250, P225, P200, P175, P150, P125, " - + "P100, P75, P50, P30, P20, P10'"), + 'var1_fcst_levels': ("'P1000, P925, P850, P700, P500, P400, " + + "P300, P250, P200, P150, P100, " + + "P50'"), 'var1_fcst_thresholds': '', 'var1_fcst_options': '', 'var1_obs_name': 'UGRD', - 'var1_obs_levels': ("'P1000, P975, P950, P925, P900, P875, " - + "P850, P825, P800, P775, P750, P725, " - + "P700, P675, P650, P625, P600, P575, " - + "P550, P525, P500, P475, P450, P425, " - + "P400, P375, P350, P325, P300, P275, " - + "P250, P225, P200, P175, P150, P125, " - + "P100, P75, P50, P30, P20, P10'"), + 'var1_obs_levels': ("'P1000, P925, P850, P700, P500, P400, " + + "P300, P250, P200, P150, P100, " + + "P50'"), 'var1_obs_thresholds': '', 'var1_obs_options': '', 'var2_fcst_name': 'VGRD', - 'var2_fcst_levels': ("'P1000, P975, P950, P925, P900, P875, " - + "P850, P825, P800, P775, P750, P725, " - + "P700, P675, P650, P625, P600, P575, " - + "P550, P525, P500, P475, P450, P425, " - + "P400, P375, P350, P325, P300, P275, " - + "P250, P225, P200, P175, P150, P125, " - + "P100, P75, P50, P30, P20, P10'"), + 'var2_fcst_levels': ("'P1000, P925, P850, P700, P500, P400, " + + "P300, P250, P200, P150, P100, " + + "P50'"), 'var2_fcst_thresholds': '', 'var2_fcst_options': '', 'var2_obs_name': 'VGRD', - 'var2_obs_levels': ("'P1000, P975, P950, P925, P900, P875, " - + "P850, P825, P800, P775, P750, P725, " - + "P700, P675, P650, P625, P600, P575, " - + "P550, P525, P500, P475, P450, P425, " - + "P400, P375, P350, P325, P300, P275, " - + "P250, P225, P200, P175, P150, P125, " - + "P100, P75, P50, P30, P20, P10'"), + 'var2_obs_levels': ("'P1000, P925, P850, P700, P500, P400, " + + "P300, P250, P200, P150, P100, " + + "P50'"), 'var2_obs_thresholds': '', 'var2_obs_options': '', }, @@ -215,45 +151,29 @@ 'raob': { 'nam': { 'var1_fcst_name': 'UGRD', - 'var1_fcst_levels': ("'P1000, P975, P950, P925, P900, P875, " - + "P850, P825, P800, P775, P750, P725, " - + "P700, P675, P650, P625, P600, P575, " - + "P550, P525, P500, P475, P450, P425, " - + "P400, P375, P350, P325, P300, P275, " - + "P250, P225, P200, P175, P150, P125, " - + "P100, P75, P50, P30, P20, P10'"), + 'var1_fcst_levels': ("'P1000, P925, P850, P700, P500, P400, " + + "P300, P250, P200, P150, P100, P50, " + + "P20, P10'"), 'var1_fcst_thresholds': '', 'var1_fcst_options': '', 'var1_obs_name': 'UGRD', - 'var1_obs_levels': ("'P1000, P975, P950, P925, P900, P875, " - + "P850, P825, P800, P775, P750, P725, " - + "P700, P675, P650, P625, P600, P575, " - + "P550, P525, P500, P475, P450, P425, " - + "P400, P375, P350, P325, P300, P275, " - + "P250, P225, P200, P175, P150, P125, " - + "P100, P75, P50, P30, P20, P10'"), + 'var1_obs_levels': ("'P1000, P925, P850, P700, P500, P400, " + + "P300, P250, P200, P150, P100, P50, " + + "P20, P10'"), 'var1_obs_thresholds': '', 'var1_obs_options': '', }, 'rap': { 'var1_fcst_name': 'UGRD', - 'var1_fcst_levels': ("'P1000, P975, P950, P925, P900, P875, " - + "P850, P825, P800, P775, P750, P725, " - + "P700, P675, P650, P625, P600, P575, " - + "P550, P525, P500, P475, P450, P425, " - + "P400, P375, P350, P325, P300, P275, " - + "P250, P225, P200, P175, P150, P125, " - + "P100, P75, P50, P30, P20, P10'"), + 'var1_fcst_levels': ("'P1000, P925, P850, P700, P500, P400, " + + "P300, P250, P200, P150, P100, " + + "P50'"), 'var1_fcst_thresholds': '', 'var1_fcst_options': '', 'var1_obs_name': 'UGRD', - 'var1_obs_levels': ("'P1000, P975, P950, P925, P900, P875, " - + "P850, P825, P800, P775, P750, P725, " - + "P700, P675, P650, P625, P600, P575, " - + "P550, P525, P500, P475, P450, P425, " - + "P400, P375, P350, P325, P300, P275, " - + "P250, P225, P200, P175, P150, P125, " - + "P100, P75, P50, P30, P20, P10'"), + 'var1_obs_levels': ("'P1000, P925, P850, P700, P500, P400, " + + "P300, P250, P200, P150, P100, " + + "P50'"), 'var1_obs_thresholds': '', 'var1_obs_options': '', }, @@ -272,45 +192,29 @@ 'raob': { 'nam': { 'var1_fcst_name': 'VGRD', - 'var1_fcst_levels': ("'P1000, P975, P950, P925, P900, P875, " - + "P850, P825, P800, P775, P750, P725, " - + "P700, P675, P650, P625, P600, P575, " - + "P550, P525, P500, P475, P450, P425, " - + "P400, P375, P350, P325, P300, P275, " - + "P250, P225, P200, P175, P150, P125, " - + "P100, P75, P50, P30, P20, P10'"), + 'var1_fcst_levels': ("'P1000, P925, P850, P700, P500, P400, " + + "P300, P250, P200, P150, P100, P50, " + + "P20, P10'"), 'var1_fcst_thresholds': '', 'var1_fcst_options': '', 'var1_obs_name': 'VGRD', - 'var1_obs_levels': ("'P1000, P975, P950, P925, P900, P875, " - + "P850, P825, P800, P775, P750, P725, " - + "P700, P675, P650, P625, P600, P575, " - + "P550, P525, P500, P475, P450, P425, " - + "P400, P375, P350, P325, P300, P275, " - + "P250, P225, P200, P175, P150, P125, " - + "P100, P75, P50, P30, P20, P10'"), + 'var1_obs_levels': ("'P1000, P925, P850, P700, P500, P400, " + + "P300, P250, P200, P150, P100, P50, " + + "P20, P10'"), 'var1_obs_thresholds': '', 'var1_obs_options': '', }, 'rap': { 'var1_fcst_name': 'VGRD', - 'var1_fcst_levels': ("'P1000, P975, P950, P925, P900, P875, " - + "P850, P825, P800, P775, P750, P725, " - + "P700, P675, P650, P625, P600, P575, " - + "P550, P525, P500, P475, P450, P425, " - + "P400, P375, P350, P325, P300, P275, " - + "P250, P225, P200, P175, P150, P125, " - + "P100, P75, P50, P30, P20, P10'"), + 'var1_fcst_levels': ("'P1000, P925, P850, P700, P500, P400, " + + "P300, P250, P200, P150, P100, " + + "P50'"), 'var1_fcst_thresholds': '', 'var1_fcst_options': '', 'var1_obs_name': 'VGRD', - 'var1_obs_levels': ("'P1000, P975, P950, P925, P900, P875, " - + "P850, P825, P800, P775, P750, P725, " - + "P700, P675, P650, P625, P600, P575, " - + "P550, P525, P500, P475, P450, P425, " - + "P400, P375, P350, P325, P300, P275, " - + "P250, P225, P200, P175, P150, P125, " - + "P100, P75, P50, P30, P20, P10'"), + 'var1_obs_levels': ("'P1000, P925, P850, P700, P500, P400, " + + "P300, P250, P200, P150, P100, " + + "P50'"), 'var1_obs_thresholds': '', 'var1_obs_options': '', }, @@ -329,45 +233,29 @@ 'raob': { 'nam': { 'var1_fcst_name': 'SPFH', - 'var1_fcst_levels': ("'P1000, P975, P950, P925, P900, P875, " - + "P850, P825, P800, P775, P750, P725, " - + "P700, P675, P650, P625, P600, P575, " - + "P550, P525, P500, P475, P450, P425, " - + "P400, P375, P350, P325, P300, P275, " - + "P250, P225, P200, P175, P150, P125, " - + "P100, P75, P50, P30, P20, P10'"), + 'var1_fcst_levels': ("'P1000, P925, P850, P700, P500, P400, " + + "P300, P250, P200, P150, P100, P50, " + + "P20, P10'"), 'var1_fcst_thresholds': '', 'var1_fcst_options': 'set_attr_units = \\"g/kg\\"; convert(x)=x*1000', 'var1_obs_name': 'SPFH', - 'var1_obs_levels': ("'P1000, P975, P950, P925, P900, P875, " - + "P850, P825, P800, P775, P750, P725, " - + "P700, P675, P650, P625, P600, P575, " - + "P550, P525, P500, P475, P450, P425, " - + "P400, P375, P350, P325, P300, P275, " - + "P250, P225, P200, P175, P150, P125, " - + "P100, P75, P50, P30, P20, P10'"), + 'var1_obs_levels': ("'P1000, P925, P850, P700, P500, P400, " + + "P300, P250, P200, P150, P100, P50, " + + "P20, P10'"), 'var1_obs_thresholds': '', 'var1_obs_options': 'set_attr_units = \\"g/kg\\"; convert(x)=x*1000', }, 'rap': { 'var1_fcst_name': 'SPFH', - 'var1_fcst_levels': ("'P1000, P975, P950, P925, P900, P875, " - + "P850, P825, P800, P775, P750, P725, " - + "P700, P675, P650, P625, P600, P575, " - + "P550, P525, P500, P475, P450, P425, " - + "P400, P375, P350, P325, P300, P275, " - + "P250, P225, P200, P175, P150, P125, " - + "P100, P75, P50, P30, P20, P10'"), + 'var1_fcst_levels': ("'P1000, P925, P850, P700, P500, P400, " + + "P300, P250, P200, P150, P100, " + + "P50'"), 'var1_fcst_thresholds': '', 'var1_fcst_options': 'set_attr_units = \\"g/kg\\"; convert(x)=x*1000', 'var1_obs_name': 'SPFH', - 'var1_obs_levels': ("'P1000, P975, P950, P925, P900, P875, " - + "P850, P825, P800, P775, P750, P725, " - + "P700, P675, P650, P625, P600, P575, " - + "P550, P525, P500, P475, P450, P425, " - + "P400, P375, P350, P325, P300, P275, " - + "P250, P225, P200, P175, P150, P125, " - + "P100, P75, P50, P30, P20, P10'"), + 'var1_obs_levels': ("'P1000, P925, P850, P700, P500, P400, " + + "P300, P250, P200, P150, P100, " + + "P50'"), 'var1_obs_thresholds': '', 'var1_obs_options': 'set_attr_units = \\"g/kg\\"; convert(x)=x*1000', }, @@ -800,7 +688,7 @@ 'var1_fcst_name': 'TCDC', 'var1_fcst_levels': 'L0', 'var1_fcst_thresholds': 'lt10,gt10,gt50,gt90', - 'var1_fcst_options': 'set_attr_level = \\"TOTAL\\";', + 'var1_fcst_options': 'set_attr_level = \\"TOTAL\\"; GRIB2_pdt=0; GRIB2_ipdtmpl_index = [ 8 ]; GRIB2_ipdtmpl_val = [ {lead?fmt=%H} ];', 'var1_obs_name': 'TCDC', 'var1_obs_levels': 'L0', 'var1_obs_thresholds': 'lt10,gt10,gt50,gt90', @@ -810,7 +698,7 @@ 'var1_fcst_name': 'TCDC', 'var1_fcst_levels': 'L0', 'var1_fcst_thresholds': 'lt10,gt10,gt50,gt90', - 'var1_fcst_options': 'set_attr_level = \\"TOTAL\\";', + 'var1_fcst_options': 'set_attr_level = \\"TOTAL\\"; GRIB2_ipdtmpl_index = [ 9 ]; GRIB2_ipdtmpl_val = [ 10 ];', 'var1_obs_name': 'TCDC', 'var1_obs_levels': 'L0', 'var1_obs_thresholds': 'lt10,gt10,gt50,gt90', diff --git a/ush/mesoscale/mesoscale_util.py b/ush/mesoscale/mesoscale_util.py index 0ef4d2d6b..570eee8e4 100644 --- a/ush/mesoscale/mesoscale_util.py +++ b/ush/mesoscale/mesoscale_util.py @@ -11,7 +11,7 @@ import os import sys -import datetime +from datetime import datetime, timedelta as td import numpy as np import glob import subprocess @@ -334,7 +334,7 @@ def format_filler(unfilled_file_format, valid_time_dt, init_time_dt, shift = (filled_file_format_chunk.partition('shift=')[2]\ .partition('}')[0]) init_shift_time_dt = ( - init_time_dt + datetime.timedelta(hours=int(shift)) + init_time_dt + td(hours=int(shift)) ) replace_format_opt_count = init_shift_time_dt.strftime( format_opt_count_fmt @@ -343,7 +343,7 @@ def format_filler(unfilled_file_format, valid_time_dt, init_time_dt, shift = (filled_file_format_chunk.partition('shift=')[2]\ .partition('}')[0]) valid_shift_time_dt = ( - valid_time_dt + datetime.timedelta(hours=int(shift)) + valid_time_dt + td(hours=int(shift)) ) replace_format_opt_count = valid_shift_time_dt.strftime( format_opt_count_fmt @@ -386,15 +386,15 @@ def initalize_job_env_dict(): 'machine', 'evs_ver', 'HOMEevs', 'FIXevs', 'USHevs', 'DATA', 'NET', 'RUN', 'VERIF_CASE', 'STEP', 'COMPONENT', 'evs_run_mode', 'COMROOT', 'COMIN', 'COMOUT', 'COMOUTsmall', 'COMOUTfinal', 'EVSIN', - 'METPLUS_PATH','LOG_MET_OUTPUT_TO_METPLUS', 'METPLUS_VERBOSITY', - 'MET_ROOT', 'MET_VERBOSITY', + 'METPLUS_PATH','LOG_MET_OUTPUT_TO_METPLUS', + 'MET_ROOT', 'MET_TMP_DIR', 'MODELNAME', 'JOB_GROUP' ] job_env_dict = {} for env_var in job_env_var_list: job_env_dict[env_var] = os.environ[env_var] if env_var in ['LOG_MET_OUTPUT_TO_METPLUS', - 'METPLUS_VERBOSITY', 'MET_VERBOSITY']: + ]: job_env_dict[env_var.lower()] = os.environ[env_var] return job_env_dict @@ -437,7 +437,7 @@ def precip_check_obs_input_output_files(job_dict): DATA_files_list - list of output DATA files (strings) """ - valid_date_dt = datetime.datetime.strptime( + valid_date_dt = datetime.strptime( job_dict['DATE']+job_dict['valid_hour_start'], '%Y%m%d%H' ) @@ -455,7 +455,7 @@ def precip_check_obs_input_output_files(job_dict): job_dict['DATA'], 'data', 'ccpa', f"ccpa.accum{job_dict['ccpa_file_accum'].zfill(2)}hr.v" +(valid_date_dt - -datetime.timedelta(hours=(n-1) + -td(hours=(n-1) *int(job_dict['ccpa_file_accum'])))\ .strftime('%Y%m%d%H') ) @@ -545,12 +545,12 @@ def precip_check_model_input_output_files(job_dict): DATA_files_list - list of output DATA files (strings) """ - valid_date_dt = datetime.datetime.strptime( + valid_date_dt = datetime.strptime( job_dict['DATE']+job_dict['valid_hour_start'], '%Y%m%d%H' ) init_date_dt = (valid_date_dt - - datetime.timedelta(hours=int(job_dict['fcst_hour']))) + - td(hours=int(job_dict['fcst_hour']))) # Expected input file input_files_list = [] if job_dict['JOB_GROUP'] == 'assemble_data': @@ -672,7 +672,7 @@ def snowfall_check_obs_input_output_files(job_dict): DATA_files_list - list of output DATA files (strings) """ - valid_date_dt = datetime.datetime.strptime( + valid_date_dt = datetime.strptime( job_dict['DATE']+job_dict['valid_hour_start'], '%Y%m%d%H' ) @@ -734,12 +734,12 @@ def snowfall_check_model_input_output_files(job_dict): DATA_files_list - list of output DATA files (strings) """ - valid_date_dt = datetime.datetime.strptime( + valid_date_dt = datetime.strptime( job_dict['DATE']+job_dict['valid_hour_start'], '%Y%m%d%H' ) init_date_dt = (valid_date_dt - - datetime.timedelta(hours=int(job_dict['fcst_hour']))) + - td(hours=int(job_dict['fcst_hour']))) # Expected input file input_files_list = [] if job_dict['JOB_GROUP'] == 'assemble_data': @@ -828,6 +828,7 @@ def snowfall_check_model_input_output_files(job_dict): all_COMOUT_file_exist, COMOUT_files_list, DATA_files_list) + def get_completed_jobs(completed_jobs_file): completed_jobs = set() if os.path.exists(completed_jobs_file): @@ -838,3 +839,111 @@ def get_completed_jobs(completed_jobs_file): def mark_job_completed(completed_jobs_file, job_name): with open(completed_jobs_file, 'a') as f: f.write(job_name + "\n") + +# Construct a file name given a template +def fname_constructor(template_str, IDATE="YYYYmmdd", IHOUR="HH", + VDATE="YYYYmmdd", VHOUR="HH", VDATEHOUR="YYYYmmddHH", + VDATEm1H="YYYYmmdd", VDATEHOURm1H="YYYYmmddHH", + FHR="HH", LVL="0", OFFSET="HH"): + template_str = template_str.replace('{IDATE}', IDATE) + template_str = template_str.replace('{IHOUR}', IHOUR) + template_str = template_str.replace('{VDATE}', VDATE) + template_str = template_str.replace('{VHOUR}', VHOUR) + template_str = template_str.replace('{VDATEHOUR}', VDATEHOUR) + template_str = template_str.replace('{VDATEm1H}', VDATEm1H) + template_str = template_str.replace('{VDATEHOURm1H}', VDATEHOURm1H) + template_str = template_str.replace('{FHR}', FHR) + template_str = template_str.replace('{LVL}', LVL) + template_str = template_str.replace('{OFFSET}', OFFSET) + return template_str + +# Create a list of prepbufr file paths +def get_prepbufr_templates(indir, vdates, paths=[], obsname='both', already_preprocessed=False): + ''' + indir - (str) Input directory for prepbufr file data + vdates - (datetime object) List of datetimes used to fill templates + paths - (list of str) list of paths to append the prepbufr paths to + Default is empty. + ''' + prepbufr_templates = [] + prepbufr_paths = [] + for v, vdate in enumerate(vdates): + vh = vdate.strftime('%H') + vd = vdate.strftime('%Y%m%d') + if vh in ['00', '03', '06', '09', '12', '15', '18', '21']: + if vh in ['03', '09', '15', '21']: + offsets = ['03'] + elif vh in ['00', '06', '12', '18']: + offsets = ['00', '06'] + if obsname in ['both', 'raob']: + if not already_preprocessed: + prepbufr_templates.append(os.path.join( + indir, + 'gdas.{VDATE}', + '{VHOUR}', + 'atmos', + 'gdas.t{VHOUR}z.prepbufr' + )) + else: + prepbufr_templates.append(os.path.join( + indir, + 'gdas.t{VHOUR}z.prepbufr' + )) + for offset in offsets: + use_vdate = vdate + td(hours=int(offset)) + use_vd = use_vdate.strftime('%Y%m%d') + use_vh = use_vdate.strftime('%H') + if obsname in ['both', 'metar']: + if not already_preprocessed: + template = os.path.join( + indir, + 'nam.{VDATE}', + 'nam.t{VHOUR}z.prepbufr.tm{OFFSET}' + ) + else: + template = os.path.join( + indir, + 'nam.t{VHOUR}z.prepbufr.tm{OFFSET}' + ) + prepbufr_paths.append(fname_constructor( + template, VDATE=use_vd, VHOUR=use_vh, OFFSET=offset + )) + for template in prepbufr_templates: + prepbufr_paths.append(fname_constructor( + template, VDATE=vd, VHOUR=vh + )) + return np.concatenate((paths, np.unique(prepbufr_paths))) + +def preprocess_prepbufr(indir, fname, workdir, outdir, subsets): + if os.path.exists(os.path.join(outdir, fname)): + print(f"{fname} exists in {outdir} so we can skip preprocessing.") + else: + wd = os.getcwd() + os.chdir(workdir) + if os.path.isfile(os.path.join(indir, fname)): + run_shell_command( + [ + os.path.join(os.environ['bufr_ROOT'], 'bin', 'split_by_subset'), + os.path.join(indir, fname) + ] + ) + if all([os.path.isfile(subset) for subset in subsets]): + run_shell_command( + np.concatenate(( + ['cat'], subsets, ['>>', os.path.join(outdir, fname)] + )) + ) + else: + raise FileNotFoundError( + f"The following prepbufr subsets do not exist in {workdir}: " + + ', '.join([subset for subset in subsets if not os.path.isfile(subset)]) + + ". Cannot concatenate subsets." + ) + else: + print( + "WARNING: The following file does not exist: " + + f"{os.path.join(indir, fname)}." + + " Skipping split by subset." + ) + os.chdir(wd) + diff --git a/ush/mesoscale/prune_stat_files.py b/ush/mesoscale/prune_stat_files.py index 1c32cb460..db2c12678 100755 --- a/ush/mesoscale/prune_stat_files.py +++ b/ush/mesoscale/prune_stat_files.py @@ -112,10 +112,10 @@ def prune_data(data_dir, prune_dir, tmp_dir, output_base_template, valid_range, ''' # Prune the MET .stat files and write to new file for met_stat_file in met_stat_files: - ps = subprocess.Popen('grep -R "'+model+'" '+met_stat_file+filter_cmd, + ps = subprocess.run('grep -R "'+model+'" '+met_stat_file+filter_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, encoding='UTF-8') - grep_output = ps.communicate()[0] + grep_output = ps.stdout all_grep_output = all_grep_output+grep_output pruned_met_stat_file = os.path.join(pruned_data_dir, model+'.stat') From 82575e094de61cf64e471ad285131349d16ef3c0 Mon Sep 17 00:00:00 2001 From: PerryShafran-NOAA <62255233+PerryShafran-NOAA@users.noreply.github.com> Date: Tue, 19 Nov 2024 13:54:34 -0500 Subject: [PATCH 3/6] Fixing bug in the ceiling processing for the perfdiag (#608) --- ush/analyses/performance_diagram.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/analyses/performance_diagram.py b/ush/analyses/performance_diagram.py index a94841536..55a728f3d 100755 --- a/ush/analyses/performance_diagram.py +++ b/ush/analyses/performance_diagram.py @@ -667,7 +667,7 @@ def plot_performance_diagram(df: pd.DataFrame, logger: logging.Logger, unit_convert = False elif str(df['OBS_VAR'].tolist()[0]).upper() in ['HGT']: unit_convert = False - if str(var_long_name_key).upper() == 'HGTCLDCEIL': + if str(var_long_name_key).upper() in ['HGTCLDCEIL', 'CEIL']: if str(df['OBS_VAR'].tolist()[0]).upper() in ['CEILING']: if units in ['m', 'gpm']: units = 'gpm' From 01ab86dfc606300c71ef8dfefab5305adbb1ba1b Mon Sep 17 00:00:00 2001 From: BinbinZhou-NOAA <67291029+BinbinZhou-NOAA@users.noreply.github.com> Date: Wed, 20 Nov 2024 10:27:42 -0500 Subject: [PATCH 4/6] Feature/sref fixes (#607) * Update CAPE settings in conf files * Update sref components * Add input file checking before running metplus * Update plot scripts * update plot scripts and ush scripts * Change to MDMP * Add cat log files * Update for MDMP * Update precip.x path in ush/mesoscale/evs_prepare_sref.sh * Fix dismatch in ecf and dev scripts * Bump up walltime for grid2obs stat job * Adjust the walltime for all plot jobs and update output graphic file namings * Fix output file name again for exevs_mesoscale_sref_cape_plots.sh * Adjust memory for jevs_mesoscale_sref_grid2obs_last90days_plots.sh --- ...s_mesoscale_sref_cape_last90days_plots.sh} | 12 +- ..._mesoscale_sref_cloud_last90days_plots.sh} | 12 +- ...vs_mesoscale_sref_cnv_last90days_plots.sh} | 12 +- ...soscale_sref_grid2obs_last90days_plots.sh} | 12 +- ...mesoscale_sref_precip_last90days_plots.sh} | 12 +- ...evs_mesoscale_sref_precip_spatial_plots.sh | 6 +- ...s_mesoscale_sref_td2m_last90days_plots.sh} | 12 +- .../jevs_mesoscale_sref_grid2obs_stats.sh | 9 +- .../jevs_mesoscale_sref_precip_stats.sh | 7 +- ecf/defs/evs-nco.def | 12 +- ..._mesoscale_sref_cape_last90days_plots.ecf} | 6 +- ...mesoscale_sref_cloud_last90days_plots.ecf} | 6 +- ...s_mesoscale_sref_cnv_last90days_plots.ecf} | 8 +- ...oscale_sref_grid2obs_last90days_plots.ecf} | 8 +- ...esoscale_sref_precip_last90days_plots.ecf} | 6 +- ...vs_mesoscale_sref_precip_spatial_plots.ecf | 2 +- ..._mesoscale_sref_td2m_last90days_plots.ecf} | 8 +- .../jevs_mesoscale_sref_grid2obs_stats.ecf | 5 +- .../jevs_mesoscale_sref_precip_stats.ecf | 4 +- .../EnsembleStat_fcstSREF_obsPREPBUFR.conf | 26 +-- .../GenEnsProd_fcstSREF_obsPREPBUFR.conf | 36 +--- .../grid2obs/Pb2nc_obsGFS_Prepbufr.conf | 30 --- .../PointStat_fcstSREF_obsPREPBUFR_cnv.conf | 30 --- .../PointStat_fcstSREF_obsPREPBUFR_mean.conf | 36 +--- .../PointStat_fcstSREF_obsPREPBUFR_prob.conf | 33 --- ...ysis_fcstSREF_obsPREPBUFR_GatherByDay.conf | 18 -- .../EnsembleStat_fcstSREF_obsCCPA_G212.conf | 24 --- .../GenEnsProd_fcstSREF_obsCCPA_G212.conf | 31 --- .../GridStat_fcstSREF_obsCCPA_mean_G212.conf | 35 ---- .../GridStat_fcstSREF_obsCCPA_mean_G240.conf | 26 --- .../GridStat_fcstSREF_obsCCPA_prob_G212.conf | 45 ---- .../precip/PcpCombine_fcstSREF_APCP06h.conf | 5 - .../precip/PcpCombine_fcstSREF_APCP24h.conf | 5 - ...tAnlysis_fcstSREF_obsCCPA_GatherByDay.conf | 18 -- .../exevs_mesoscale_sref_cape_plots.sh | 91 +++++---- .../exevs_mesoscale_sref_cloud_plots.sh | 91 +++++---- .../exevs_mesoscale_sref_cnv_plots.sh | 90 ++++---- .../exevs_mesoscale_sref_grid2obs_plots.sh | 96 +++++---- .../exevs_mesoscale_sref_precip_plots.sh | 84 +++++--- ...evs_mesoscale_sref_precip_spatial_plots.sh | 6 +- .../exevs_mesoscale_sref_td2m_plots.sh | 90 ++++---- .../exevs_mesoscale_sref_grid2obs_stats.sh | 1 + .../exevs_mesoscale_sref_precip_stats.sh | 1 + ush/mesoscale/evs_prepare_sref.sh | 66 ++++-- ush/mesoscale/evs_sref_average_cnv.sh | 19 +- ush/mesoscale/evs_sref_cnv.sh | 120 +++++------ ush/mesoscale/evs_sref_gather.sh | 23 +-- ush/mesoscale/evs_sref_grid2obs.sh | 170 +++++++-------- ush/mesoscale/evs_sref_precip.sh | 193 ++++++++---------- .../ush_sref_plot_cnv_py/prune_stat_files.py | 8 +- .../prune_stat_files.py | 8 +- .../sref_atmos_util.py | 11 +- .../ush_sref_plot_py/prune_stat_files.py | 8 +- 53 files changed, 683 insertions(+), 1050 deletions(-) rename dev/drivers/scripts/plots/mesoscale/{jevs_mesoscale_sref_cape_past90days_plots.sh => jevs_mesoscale_sref_cape_last90days_plots.sh} (86%) rename dev/drivers/scripts/plots/mesoscale/{jevs_mesoscale_sref_cloud_past90days_plots.sh => jevs_mesoscale_sref_cloud_last90days_plots.sh} (86%) rename dev/drivers/scripts/plots/mesoscale/{jevs_mesoscale_sref_cnv_past90days_plots.sh => jevs_mesoscale_sref_cnv_last90days_plots.sh} (86%) rename dev/drivers/scripts/plots/mesoscale/{jevs_mesoscale_sref_grid2obs_past90days_plots.sh => jevs_mesoscale_sref_grid2obs_last90days_plots.sh} (86%) rename dev/drivers/scripts/plots/mesoscale/{jevs_mesoscale_sref_precip_past90days_plots.sh => jevs_mesoscale_sref_precip_last90days_plots.sh} (86%) rename dev/drivers/scripts/plots/mesoscale/{jevs_mesoscale_sref_td2m_past90days_plots.sh => jevs_mesoscale_sref_td2m_last90days_plots.sh} (86%) rename ecf/scripts/plots/mesoscale/{jevs_mesoscale_sref_cape_past90days_plots.ecf => jevs_mesoscale_sref_cape_last90days_plots.ecf} (93%) rename ecf/scripts/plots/mesoscale/{jevs_mesoscale_sref_cloud_past90days_plots.ecf => jevs_mesoscale_sref_cloud_last90days_plots.ecf} (93%) rename ecf/scripts/plots/mesoscale/{jevs_mesoscale_sref_td2m_past90days_plots.ecf => jevs_mesoscale_sref_cnv_last90days_plots.ecf} (92%) rename ecf/scripts/plots/mesoscale/{jevs_mesoscale_sref_grid2obs_past90days_plots.ecf => jevs_mesoscale_sref_grid2obs_last90days_plots.ecf} (91%) rename ecf/scripts/plots/mesoscale/{jevs_mesoscale_sref_precip_past90days_plots.ecf => jevs_mesoscale_sref_precip_last90days_plots.ecf} (93%) rename ecf/scripts/plots/mesoscale/{jevs_mesoscale_sref_cnv_past90days_plots.ecf => jevs_mesoscale_sref_td2m_last90days_plots.ecf} (92%) diff --git a/dev/drivers/scripts/plots/mesoscale/jevs_mesoscale_sref_cape_past90days_plots.sh b/dev/drivers/scripts/plots/mesoscale/jevs_mesoscale_sref_cape_last90days_plots.sh similarity index 86% rename from dev/drivers/scripts/plots/mesoscale/jevs_mesoscale_sref_cape_past90days_plots.sh rename to dev/drivers/scripts/plots/mesoscale/jevs_mesoscale_sref_cape_last90days_plots.sh index f42a6caad..979eeedb3 100755 --- a/dev/drivers/scripts/plots/mesoscale/jevs_mesoscale_sref_cape_past90days_plots.sh +++ b/dev/drivers/scripts/plots/mesoscale/jevs_mesoscale_sref_cape_last90days_plots.sh @@ -1,23 +1,21 @@ -#PBS -N jevs_mesoscale_sref_cape_past90days_plots +#PBS -N jevs_mesoscale_sref_cape_last90days_plots #PBS -j oe #PBS -S /bin/bash #PBS -q dev #PBS -A VERF-DEV -#PBS -l walltime=00:30:00 -#PBS -l place=vscatter,select=1:ncpus=80:mem=300GB +#PBS -l walltime=00:10:00 +#PBS -l place=vscatter,select=1:ncpus=80:mem=40GB #PBS -l debug=true set -x export OMP_NUM_THREADS=1 +export NET=evs export HOMEevs=/lfs/h2/emc/vpppg/noscrub/${USER}/EVS - source $HOMEevs/versions/run.ver export envir=prod - -export NET=evs export STEP=plots export COMPONENT=mesoscale export RUN=atmos @@ -33,7 +31,7 @@ export KEEPDATA=YES export SENDMAIL=YES export SENDDBN=NO -export past_days=90 +export last_days=90 export run_mpi=yes export valid_time=both diff --git a/dev/drivers/scripts/plots/mesoscale/jevs_mesoscale_sref_cloud_past90days_plots.sh b/dev/drivers/scripts/plots/mesoscale/jevs_mesoscale_sref_cloud_last90days_plots.sh similarity index 86% rename from dev/drivers/scripts/plots/mesoscale/jevs_mesoscale_sref_cloud_past90days_plots.sh rename to dev/drivers/scripts/plots/mesoscale/jevs_mesoscale_sref_cloud_last90days_plots.sh index 11647cdb6..3805ad0f7 100755 --- a/dev/drivers/scripts/plots/mesoscale/jevs_mesoscale_sref_cloud_past90days_plots.sh +++ b/dev/drivers/scripts/plots/mesoscale/jevs_mesoscale_sref_cloud_last90days_plots.sh @@ -1,23 +1,21 @@ -#PBS -N jevs_mesoscale_sref_cloud_past90days_plots +#PBS -N jevs_mesoscale_sref_cloud_last90days_plots #PBS -j oe #PBS -S /bin/bash #PBS -q dev #PBS -A VERF-DEV -#PBS -l walltime=00:30:00 -#PBS -l place=vscatter,select=1:ncpus=112:mem=300GB +#PBS -l walltime=00:15:00 +#PBS -l place=vscatter,select=1:ncpus=112:mem=100GB #PBS -l debug=true set -x export OMP_NUM_THREADS=1 +export NET=evs export HOMEevs=/lfs/h2/emc/vpppg/noscrub/${USER}/EVS - source $HOMEevs/versions/run.ver export envir=prod - -export NET=evs export STEP=plots export COMPONENT=mesoscale export RUN=atmos @@ -33,7 +31,7 @@ export KEEPDATA=YES export SENDMAIL=YES export SENDDBN=NO -export past_days=90 +export last_days=90 export run_mpi=yes export valid_time=both diff --git a/dev/drivers/scripts/plots/mesoscale/jevs_mesoscale_sref_cnv_past90days_plots.sh b/dev/drivers/scripts/plots/mesoscale/jevs_mesoscale_sref_cnv_last90days_plots.sh similarity index 86% rename from dev/drivers/scripts/plots/mesoscale/jevs_mesoscale_sref_cnv_past90days_plots.sh rename to dev/drivers/scripts/plots/mesoscale/jevs_mesoscale_sref_cnv_last90days_plots.sh index 081dcb783..2b12092db 100755 --- a/dev/drivers/scripts/plots/mesoscale/jevs_mesoscale_sref_cnv_past90days_plots.sh +++ b/dev/drivers/scripts/plots/mesoscale/jevs_mesoscale_sref_cnv_last90days_plots.sh @@ -1,23 +1,21 @@ -#PBS -N jevs_mesoscale_sref_cnv_past90days_plots +#PBS -N jevs_mesoscale_sref_cnv_last90days_plots #PBS -j oe #PBS -S /bin/bash #PBS -q dev #PBS -A VERF-DEV -#PBS -l walltime=00:30:00 -#PBS -l place=vscatter,select=2:ncpus=88:mem=300GB +#PBS -l walltime=00:10:00 +#PBS -l place=vscatter,select=2:ncpus=88:mem=20GB #PBS -l debug=true set -x export OMP_NUM_THREADS=1 +export NET=evs export HOMEevs=/lfs/h2/emc/vpppg/noscrub/${USER}/EVS - source $HOMEevs/versions/run.ver export envir=prod - -export NET=evs export STEP=plots export COMPONENT=mesoscale export RUN=atmos @@ -33,7 +31,7 @@ export KEEPDATA=YES export SENDMAIL=YES export SENDDBN=NO -export past_days=90 +export last_days=90 export run_mpi=yes export valid_time=both diff --git a/dev/drivers/scripts/plots/mesoscale/jevs_mesoscale_sref_grid2obs_past90days_plots.sh b/dev/drivers/scripts/plots/mesoscale/jevs_mesoscale_sref_grid2obs_last90days_plots.sh similarity index 86% rename from dev/drivers/scripts/plots/mesoscale/jevs_mesoscale_sref_grid2obs_past90days_plots.sh rename to dev/drivers/scripts/plots/mesoscale/jevs_mesoscale_sref_grid2obs_last90days_plots.sh index 321b6a5b0..9932bc2ce 100755 --- a/dev/drivers/scripts/plots/mesoscale/jevs_mesoscale_sref_grid2obs_past90days_plots.sh +++ b/dev/drivers/scripts/plots/mesoscale/jevs_mesoscale_sref_grid2obs_last90days_plots.sh @@ -1,23 +1,21 @@ -#PBS -N jevs_mesoscale_sref_grid2obs_past90days_plots +#PBS -N jevs_mesoscale_sref_grid2obs_last90days_plots #PBS -j oe #PBS -S /bin/bash #PBS -q dev #PBS -A VERF-DEV -#PBS -l walltime=00:30:00 -#PBS -l place=vscatter,select=3:ncpus=72:mem=300GB +#PBS -l walltime=00:10:00 +#PBS -l place=vscatter,select=3:ncpus=72:mem=100GB #PBS -l debug=true set -x export OMP_NUM_THREADS=1 +export NET=evs export HOMEevs=/lfs/h2/emc/vpppg/noscrub/${USER}/EVS - source $HOMEevs/versions/run.ver export envir=prod - -export NET=evs export STEP=plots export COMPONENT=mesoscale export RUN=atmos @@ -33,7 +31,7 @@ export KEEPDATA=YES export SENDMAIL=YES export SENDDBN=NO -export past_days=90 +export last_days=90 export run_mpi=yes export valid_time=both diff --git a/dev/drivers/scripts/plots/mesoscale/jevs_mesoscale_sref_precip_past90days_plots.sh b/dev/drivers/scripts/plots/mesoscale/jevs_mesoscale_sref_precip_last90days_plots.sh similarity index 86% rename from dev/drivers/scripts/plots/mesoscale/jevs_mesoscale_sref_precip_past90days_plots.sh rename to dev/drivers/scripts/plots/mesoscale/jevs_mesoscale_sref_precip_last90days_plots.sh index 0b873cc8a..a330607d9 100755 --- a/dev/drivers/scripts/plots/mesoscale/jevs_mesoscale_sref_precip_past90days_plots.sh +++ b/dev/drivers/scripts/plots/mesoscale/jevs_mesoscale_sref_precip_last90days_plots.sh @@ -1,23 +1,21 @@ -#PBS -N jevs_mesoscale_sref_precip_past90days_plots +#PBS -N jevs_mesoscale_sref_precip_last90days_plots #PBS -j oe #PBS -S /bin/bash #PBS -q dev #PBS -A VERF-DEV -#PBS -l walltime=00:30:00 -#PBS -l place=vscatter,select=1:ncpus=100:mem=300GB +#PBS -l walltime=00:15:00 +#PBS -l place=vscatter,select=1:ncpus=100:mem=100GB #PBS -l debug=true set -x export OMP_NUM_THREADS=1 +export NET=evs export HOMEevs=/lfs/h2/emc/vpppg/noscrub/${USER}/EVS - source $HOMEevs/versions/run.ver export envir=prod - -export NET=evs export STEP=plots export COMPONENT=mesoscale export RUN=atmos @@ -33,7 +31,7 @@ export KEEPDATA=YES export SENDMAIL=YES export SENDDBN=NO -export past_days=90 +export last_days=90 export run_mpi=yes export valid_time=both diff --git a/dev/drivers/scripts/plots/mesoscale/jevs_mesoscale_sref_precip_spatial_plots.sh b/dev/drivers/scripts/plots/mesoscale/jevs_mesoscale_sref_precip_spatial_plots.sh index 4ba3635af..106472f18 100755 --- a/dev/drivers/scripts/plots/mesoscale/jevs_mesoscale_sref_precip_spatial_plots.sh +++ b/dev/drivers/scripts/plots/mesoscale/jevs_mesoscale_sref_precip_spatial_plots.sh @@ -4,20 +4,18 @@ #PBS -q dev #PBS -A VERF-DEV #PBS -l walltime=00:10:00 -#PBS -l place=vscatter,select=1:ncpus=2:mem=100GB +#PBS -l place=vscatter,select=1:ncpus=1:mem=10GB #PBS -l debug=true set -x export OMP_NUM_THREADS=1 +export NET=evs export HOMEevs=/lfs/h2/emc/vpppg/noscrub/${USER}/EVS - source $HOMEevs/versions/run.ver export envir=prod - -export NET=evs export STEP=plots export COMPONENT=mesoscale export RUN=atmos diff --git a/dev/drivers/scripts/plots/mesoscale/jevs_mesoscale_sref_td2m_past90days_plots.sh b/dev/drivers/scripts/plots/mesoscale/jevs_mesoscale_sref_td2m_last90days_plots.sh similarity index 86% rename from dev/drivers/scripts/plots/mesoscale/jevs_mesoscale_sref_td2m_past90days_plots.sh rename to dev/drivers/scripts/plots/mesoscale/jevs_mesoscale_sref_td2m_last90days_plots.sh index 1528944de..53092f160 100755 --- a/dev/drivers/scripts/plots/mesoscale/jevs_mesoscale_sref_td2m_past90days_plots.sh +++ b/dev/drivers/scripts/plots/mesoscale/jevs_mesoscale_sref_td2m_last90days_plots.sh @@ -1,23 +1,21 @@ -#PBS -N jevs_mesoscale_sref_td2m_past90days_plots +#PBS -N jevs_mesoscale_sref_td2m_last90days_plots #PBS -j oe #PBS -S /bin/bash #PBS -q dev #PBS -A VERF-DEV -#PBS -l walltime=00:30:00 -#PBS -l place=vscatter,select=1:ncpus=80:mem=300GB +#PBS -l walltime=00:15:00 +#PBS -l place=vscatter,select=1:ncpus=80:mem=80GB #PBS -l debug=true set -x export OMP_NUM_THREADS=1 +export NET=evs export HOMEevs=/lfs/h2/emc/vpppg/noscrub/${USER}/EVS - source $HOMEevs/versions/run.ver export envir=prod - -export NET=evs export STEP=plots export COMPONENT=mesoscale export RUN=atmos @@ -33,7 +31,7 @@ export KEEPDATA=YES export SENDMAIL=YES export SENDDBN=NO -export past_days=90 +export last_days=90 export run_mpi=yes export valid_time=both diff --git a/dev/drivers/scripts/stats/mesoscale/jevs_mesoscale_sref_grid2obs_stats.sh b/dev/drivers/scripts/stats/mesoscale/jevs_mesoscale_sref_grid2obs_stats.sh index ed3b7b6b4..5c330ae8d 100755 --- a/dev/drivers/scripts/stats/mesoscale/jevs_mesoscale_sref_grid2obs_stats.sh +++ b/dev/drivers/scripts/stats/mesoscale/jevs_mesoscale_sref_grid2obs_stats.sh @@ -3,21 +3,18 @@ #PBS -S /bin/bash #PBS -q dev #PBS -A VERF-DEV -#PBS -l walltime=02:00:00 -#PBS -l place=vscatter,select=1:ncpus=30:mem=100GB +#PBS -l walltime=00:35:00 +#PBS -l place=vscatter,select=1:ncpus=15:mem=100GB #PBS -l debug=true - set -x -#Total 2 parallel processes - export OMP_NUM_THREADS=1 +export NET=evs export HOMEevs=/lfs/h2/emc/vpppg/noscrub/${USER}/EVS source $HOMEevs/versions/run.ver export envir=prod -export NET=evs export STEP=stats export COMPONENT=mesoscale export RUN=atmos diff --git a/dev/drivers/scripts/stats/mesoscale/jevs_mesoscale_sref_precip_stats.sh b/dev/drivers/scripts/stats/mesoscale/jevs_mesoscale_sref_precip_stats.sh index eb2578eb6..3c4c7cf42 100755 --- a/dev/drivers/scripts/stats/mesoscale/jevs_mesoscale_sref_precip_stats.sh +++ b/dev/drivers/scripts/stats/mesoscale/jevs_mesoscale_sref_precip_stats.sh @@ -3,19 +3,18 @@ #PBS -S /bin/bash #PBS -q dev #PBS -A VERF-DEV -#PBS -l walltime=02:30:00 -#PBS -l place=vscatter,select=1:ncpus=8:mem=100GB +#PBS -l walltime=02:00:00 +#PBS -l place=vscatter,select=1:ncpus=28:mem=30GB #PBS -l debug=true set -x export OMP_NUM_THREADS=1 -#Total 2 processes +export NET=evs export HOMEevs=/lfs/h2/emc/vpppg/noscrub/${USER}/EVS source $HOMEevs/versions/run.ver export envir=prod -export NET=evs export STEP=stats export COMPONENT=mesoscale export RUN=atmos diff --git a/ecf/defs/evs-nco.def b/ecf/defs/evs-nco.def index 2fe92c10b..086800299 100644 --- a/ecf/defs/evs-nco.def +++ b/ecf/defs/evs-nco.def @@ -1591,17 +1591,17 @@ suite evs_nco trigger :TIME >= 1530 and ../../stats/nwps == complete endfamily family mesoscale - task jevs_mesoscale_sref_grid2obs_past90days_plots + task jevs_mesoscale_sref_grid2obs_last90days_plots trigger :TIME >= 1300 and ../../../../06/evs/stats/mesoscale/jevs_mesoscale_sref_grid2obs_stats == complete and ../../../../06/evs/stats/global_ens/jevs_global_ens_gefs_atmos_grid2obs_stats == complete - task jevs_mesoscale_sref_precip_past90days_plots + task jevs_mesoscale_sref_precip_last90days_plots trigger :TIME >= 1300 and ../../../../06/evs/stats/mesoscale/jevs_mesoscale_sref_precip_stats == complete and ../../../../06/evs/stats/global_ens/jevs_global_ens_gefs_atmos_precip_stats == complete - task jevs_mesoscale_sref_cnv_past90days_plots + task jevs_mesoscale_sref_cnv_last90days_plots trigger :TIME >= 1300 and ../../../../06/evs/stats/mesoscale/jevs_mesoscale_sref_grid2obs_stats == complete and ../../../../06/evs/stats/global_ens/jevs_global_ens_gefs_atmos_cnv_stats == complete - task jevs_mesoscale_sref_cape_past90days_plots + task jevs_mesoscale_sref_cape_last90days_plots trigger :TIME >= 1300 and ../../../../06/evs/stats/mesoscale/jevs_mesoscale_sref_grid2obs_stats == complete and ../../../../06/evs/stats/global_ens/jevs_global_ens_gefs_atmos_grid2obs_stats == complete - task jevs_mesoscale_sref_cloud_past90days_plots + task jevs_mesoscale_sref_cloud_last90days_plots trigger :TIME >= 1300 and ../../../../06/evs/stats/mesoscale/jevs_mesoscale_sref_grid2obs_stats == complete and ../../../../06/evs/stats/global_ens/jevs_global_ens_gefs_atmos_grid2obs_stats == complete - task jevs_mesoscale_sref_td2m_past90days_plots + task jevs_mesoscale_sref_td2m_last90days_plots trigger :TIME >= 1300 and ../../../../06/evs/stats/mesoscale/jevs_mesoscale_sref_grid2obs_stats == complete and ../../../../06/evs/stats/global_ens/jevs_global_ens_gefs_atmos_grid2obs_stats == complete task jevs_mesoscale_sref_precip_spatial_plots trigger :TIME >= 1300 and ../../../../06/evs/stats/mesoscale/jevs_mesoscale_sref_precip_stats == complete and ../../../../06/evs/stats/global_ens/jevs_global_ens_gefs_atmos_precip_stats == complete diff --git a/ecf/scripts/plots/mesoscale/jevs_mesoscale_sref_cape_past90days_plots.ecf b/ecf/scripts/plots/mesoscale/jevs_mesoscale_sref_cape_last90days_plots.ecf similarity index 93% rename from ecf/scripts/plots/mesoscale/jevs_mesoscale_sref_cape_past90days_plots.ecf rename to ecf/scripts/plots/mesoscale/jevs_mesoscale_sref_cape_last90days_plots.ecf index 15a029a38..10885773c 100755 --- a/ecf/scripts/plots/mesoscale/jevs_mesoscale_sref_cape_past90days_plots.ecf +++ b/ecf/scripts/plots/mesoscale/jevs_mesoscale_sref_cape_last90days_plots.ecf @@ -1,10 +1,10 @@ -#PBS -N evs_mesoscale_sref_cape_past90days_plots +#PBS -N evs_mesoscale_sref_cape_last90days_plots #PBS -j oe #PBS -S /bin/bash #PBS -q %QUEUE% #PBS -A %PROJ%-%PROJENVIR% #PBS -l walltime=00:10:00 -#PBS -l place=vscatter:exclhost,select=1:ncpus=80:mem=300GB +#PBS -l place=vscatter:exclhost,select=1:ncpus=80:mem=40GB #PBS -l debug=true export model=evs @@ -55,7 +55,7 @@ export NET=evs export RUN=atmos export VERIF_CASE=cape export MODELNAME=sref -export past_days=90 +export last_days=90 export run_mpi=yes export valid_time=both diff --git a/ecf/scripts/plots/mesoscale/jevs_mesoscale_sref_cloud_past90days_plots.ecf b/ecf/scripts/plots/mesoscale/jevs_mesoscale_sref_cloud_last90days_plots.ecf similarity index 93% rename from ecf/scripts/plots/mesoscale/jevs_mesoscale_sref_cloud_past90days_plots.ecf rename to ecf/scripts/plots/mesoscale/jevs_mesoscale_sref_cloud_last90days_plots.ecf index ee900aefb..f14c0a973 100755 --- a/ecf/scripts/plots/mesoscale/jevs_mesoscale_sref_cloud_past90days_plots.ecf +++ b/ecf/scripts/plots/mesoscale/jevs_mesoscale_sref_cloud_last90days_plots.ecf @@ -1,10 +1,10 @@ -#PBS -N evs_mesoscale_sref_cloud_past90days_plots +#PBS -N evs_mesoscale_sref_cloud_last90days_plots #PBS -j oe #PBS -S /bin/bash #PBS -q %QUEUE% #PBS -A %PROJ%-%PROJENVIR% #PBS -l walltime=00:15:00 -#PBS -l place=vscatter:exclhost,select=1:ncpus=112:mem=300GB +#PBS -l place=vscatter:exclhost,select=1:ncpus=112:mem=100GB #PBS -l debug=true export model=evs @@ -55,7 +55,7 @@ export NET=evs export RUN=atmos export VERIF_CASE=cloud export MODELNAME=sref -export past_days=90 +export last_days=90 export run_mpi=yes export valid_time=both diff --git a/ecf/scripts/plots/mesoscale/jevs_mesoscale_sref_td2m_past90days_plots.ecf b/ecf/scripts/plots/mesoscale/jevs_mesoscale_sref_cnv_last90days_plots.ecf similarity index 92% rename from ecf/scripts/plots/mesoscale/jevs_mesoscale_sref_td2m_past90days_plots.ecf rename to ecf/scripts/plots/mesoscale/jevs_mesoscale_sref_cnv_last90days_plots.ecf index 7c0735f28..89bce549e 100755 --- a/ecf/scripts/plots/mesoscale/jevs_mesoscale_sref_td2m_past90days_plots.ecf +++ b/ecf/scripts/plots/mesoscale/jevs_mesoscale_sref_cnv_last90days_plots.ecf @@ -1,10 +1,10 @@ -#PBS -N evs_mesoscale_sref_td2m_past90days_plots +#PBS -N evs_mesoscale_sref_cnv_last90days_plots #PBS -j oe #PBS -S /bin/bash #PBS -q %QUEUE% #PBS -A %PROJ%-%PROJENVIR% #PBS -l walltime=00:10:00 -#PBS -l place=vscatter:exclhost,select=1:ncpus=80:mem=300GB +#PBS -l place=vscatter:exclhost,select=2:ncpus=88:mem=20GB #PBS -l debug=true export model=evs @@ -53,9 +53,9 @@ fi export OMP_NUM_THREADS=1 export NET=evs export RUN=atmos -export VERIF_CASE=td2m +export VERIF_CASE=cnv export MODELNAME=sref -export past_days=90 +export last_days=90 export run_mpi=yes export valid_time=both diff --git a/ecf/scripts/plots/mesoscale/jevs_mesoscale_sref_grid2obs_past90days_plots.ecf b/ecf/scripts/plots/mesoscale/jevs_mesoscale_sref_grid2obs_last90days_plots.ecf similarity index 91% rename from ecf/scripts/plots/mesoscale/jevs_mesoscale_sref_grid2obs_past90days_plots.ecf rename to ecf/scripts/plots/mesoscale/jevs_mesoscale_sref_grid2obs_last90days_plots.ecf index 798dba959..6d0acb256 100755 --- a/ecf/scripts/plots/mesoscale/jevs_mesoscale_sref_grid2obs_past90days_plots.ecf +++ b/ecf/scripts/plots/mesoscale/jevs_mesoscale_sref_grid2obs_last90days_plots.ecf @@ -1,10 +1,10 @@ -#PBS -N evs_mesoscale_sref_grid2obs_past90days_plots +#PBS -N evs_mesoscale_sref_grid2obs_last90days_plots #PBS -j oe #PBS -S /bin/bash #PBS -q %QUEUE% #PBS -A %PROJ%-%PROJENVIR% -#PBS -l walltime=00:15:00 -#PBS -l place=vscatter:exclhost,select=3:ncpus=72:mem=300GB +#PBS -l walltime=00:10:00 +#PBS -l place=vscatter:exclhost,select=3:ncpus=72:mem=100GB #PBS -l debug=true export model=evs @@ -55,7 +55,7 @@ export NET=evs export RUN=atmos export VERIF_CASE=grid2obs export MODELNAME=sref -export past_days=90 +export last_days=90 export run_mpi=yes export valid_time=both diff --git a/ecf/scripts/plots/mesoscale/jevs_mesoscale_sref_precip_past90days_plots.ecf b/ecf/scripts/plots/mesoscale/jevs_mesoscale_sref_precip_last90days_plots.ecf similarity index 93% rename from ecf/scripts/plots/mesoscale/jevs_mesoscale_sref_precip_past90days_plots.ecf rename to ecf/scripts/plots/mesoscale/jevs_mesoscale_sref_precip_last90days_plots.ecf index 596b31ea3..e74a7a841 100755 --- a/ecf/scripts/plots/mesoscale/jevs_mesoscale_sref_precip_past90days_plots.ecf +++ b/ecf/scripts/plots/mesoscale/jevs_mesoscale_sref_precip_last90days_plots.ecf @@ -1,10 +1,10 @@ -#PBS -N evs_mesoscale_sref_precip_past90days_plots +#PBS -N evs_mesoscale_sref_precip_last90days_plots #PBS -j oe #PBS -S /bin/bash #PBS -q %QUEUE% #PBS -A %PROJ%-%PROJENVIR% #PBS -l walltime=00:15:00 -#PBS -l place=vscatter:exclhost,select=1:ncpus=100:mem=300GB +#PBS -l place=vscatter:exclhost,select=1:ncpus=100:mem=100GB #PBS -l debug=true export model=evs @@ -55,7 +55,7 @@ export NET=evs export RUN=atmos export VERIF_CASE=precip export MODELNAME=sref -export past_days=90 +export last_days=90 export run_mpi=yes export valid_time=both diff --git a/ecf/scripts/plots/mesoscale/jevs_mesoscale_sref_precip_spatial_plots.ecf b/ecf/scripts/plots/mesoscale/jevs_mesoscale_sref_precip_spatial_plots.ecf index d683ec6de..ef6162770 100755 --- a/ecf/scripts/plots/mesoscale/jevs_mesoscale_sref_precip_spatial_plots.ecf +++ b/ecf/scripts/plots/mesoscale/jevs_mesoscale_sref_precip_spatial_plots.ecf @@ -4,7 +4,7 @@ #PBS -q %QUEUE% #PBS -A %PROJ%-%PROJENVIR% #PBS -l walltime=00:10:00 -#PBS -l place=vscatter:shared,select=1:ncpus=2:mem=100GB +#PBS -l place=vscatter:shared,select=1:ncpus=1:mem=10GB #PBS -l debug=true export model=evs diff --git a/ecf/scripts/plots/mesoscale/jevs_mesoscale_sref_cnv_past90days_plots.ecf b/ecf/scripts/plots/mesoscale/jevs_mesoscale_sref_td2m_last90days_plots.ecf similarity index 92% rename from ecf/scripts/plots/mesoscale/jevs_mesoscale_sref_cnv_past90days_plots.ecf rename to ecf/scripts/plots/mesoscale/jevs_mesoscale_sref_td2m_last90days_plots.ecf index 2e93253e5..04ab5f696 100755 --- a/ecf/scripts/plots/mesoscale/jevs_mesoscale_sref_cnv_past90days_plots.ecf +++ b/ecf/scripts/plots/mesoscale/jevs_mesoscale_sref_td2m_last90days_plots.ecf @@ -1,10 +1,10 @@ -#PBS -N evs_mesoscale_sref_cnv_past90days_plots +#PBS -N evs_mesoscale_sref_td2m_last90days_plots #PBS -j oe #PBS -S /bin/bash #PBS -q %QUEUE% #PBS -A %PROJ%-%PROJENVIR% #PBS -l walltime=00:15:00 -#PBS -l place=vscatter:exclhost,select=2:ncpus=88:mem=300GB +#PBS -l place=vscatter:exclhost,select=1:ncpus=80:mem=80GB #PBS -l debug=true export model=evs @@ -53,9 +53,9 @@ fi export OMP_NUM_THREADS=1 export NET=evs export RUN=atmos -export VERIF_CASE=cnv +export VERIF_CASE=td2m export MODELNAME=sref -export past_days=90 +export last_days=90 export run_mpi=yes export valid_time=both diff --git a/ecf/scripts/stats/mesoscale/jevs_mesoscale_sref_grid2obs_stats.ecf b/ecf/scripts/stats/mesoscale/jevs_mesoscale_sref_grid2obs_stats.ecf index 7aaf7b857..d9d77b0e5 100755 --- a/ecf/scripts/stats/mesoscale/jevs_mesoscale_sref_grid2obs_stats.ecf +++ b/ecf/scripts/stats/mesoscale/jevs_mesoscale_sref_grid2obs_stats.ecf @@ -3,9 +3,8 @@ #PBS -S /bin/bash #PBS -q %QUEUE% #PBS -A %PROJ%-%PROJENVIR% -#PBS -l walltime=01:40:00 -##PBS -l walltime=01:25:00 -#PBS -l place=vscatter:shared,select=1:ncpus=4:mem=100GB +#PBS -l walltime=00:35:00 +#PBS -l place=vscatter,select=1:ncpus=15:mem=100GB #PBS -l debug=true export model=evs diff --git a/ecf/scripts/stats/mesoscale/jevs_mesoscale_sref_precip_stats.ecf b/ecf/scripts/stats/mesoscale/jevs_mesoscale_sref_precip_stats.ecf index d31bdd27d..a394272cd 100755 --- a/ecf/scripts/stats/mesoscale/jevs_mesoscale_sref_precip_stats.ecf +++ b/ecf/scripts/stats/mesoscale/jevs_mesoscale_sref_precip_stats.ecf @@ -3,8 +3,8 @@ #PBS -S /bin/bash #PBS -q %QUEUE% #PBS -A %PROJ%-%PROJENVIR% -#PBS -l walltime=02:15:00 -#PBS -l place=vscatter:shared,select=1:ncpus=8:mem=100GB +#PBS -l walltime=02:00:00 +#PBS -l place=vscatter,select=1:ncpus=28:mem=30GB #PBS -l debug=true export model=evs diff --git a/parm/metplus_config/stats/mesoscale/grid2obs/EnsembleStat_fcstSREF_obsPREPBUFR.conf b/parm/metplus_config/stats/mesoscale/grid2obs/EnsembleStat_fcstSREF_obsPREPBUFR.conf index a6e21dbc4..523b9cb92 100755 --- a/parm/metplus_config/stats/mesoscale/grid2obs/EnsembleStat_fcstSREF_obsPREPBUFR.conf +++ b/parm/metplus_config/stats/mesoscale/grid2obs/EnsembleStat_fcstSREF_obsPREPBUFR.conf @@ -1,8 +1,6 @@ [config] OUTPUT_BASE = {ENV[output_base]} -LOG_LINE_FORMAT = - # time looping - options are INIT, VALID, RETRO, and REALTIME LOOP_BY = VALID @@ -26,27 +24,12 @@ METPLUS_PATH = {ENV[METPLUS_PATH]} # Increment between METplus runs in seconds. Must be >= 60 # 86400 sec=24h -#VALID_INCREMENT = 21600 VALID_INCREMENT = {ENV[valid_increment]} -# Options are times, processes -# times = run all items in the PROCESS_LIST for a single initialization -# time, then repeat until all times have been evaluated. -# processes = run each item in the PROCESS_LIST for all times -# specified, then repeat for the next item in the PROCESS_LIST. -LOOP_ORDER = times # List of applications to run -#PROCESS_LIST = EnsembleStat, GridStat PROCESS_LIST = EnsembleStat -#GRID_STAT_REGRID_TO_GRID = {ENV[regrid]} -# run pcp_combine on forecast/obs data? -#FCST_PCP_COMBINE_RUN = True -#OBS_PCP_COMBINE_RUN = True -#OBS_REGRID_DATA_PLANE_RUN = True - - # list of variables to compare FCST_VAR1_NAME = TMP @@ -95,9 +78,10 @@ OBS_VAR8_LEVELS = {FCST_VAR8_LEVELS} FCST_VAR9_NAME = CAPE FCST_VAR9_LEVELS = L0 -FCST_VAR9_OPTIONS = GRIB_lvl_typ = 1 +FCST_VAR9_OPTIONS = GRIB_lvl_typ = 1; cnt_thresh = [ NA ]; cnt_logic = INTERSECTION; OBS_VAR9_NAME = {FCST_VAR9_NAME} OBS_VAR9_LEVELS = {FCST_VAR9_LEVELS} +OBS_VAR9_OPTIONS = cnt_thresh = [ >0 ]; cnt_logic = INTERSECTION; FCST_VAR10_NAME = PRMSL FCST_VAR10_LEVELS = L0 @@ -182,9 +166,6 @@ ENSEMBLE_STAT_DUPLICATE_FLAG = NONE ENSEMBLE_STAT_SKIP_CONST = False ENSEMBLE_STAT_OBS_ERROR_FLAG = FALSE -#OBS_WINDOW_BEGIN = -3600 -#OBS_WINDOW_END = 3600 - ENSEMBLE_STAT_SSVAR_BIN_SIZE = 1000000 ENSEMBLE_STAT_PHIST_BIN_SIZE = 0.05 @@ -244,9 +225,6 @@ OBS_ENSEMBLE_STAT_INPUT_POINT_DATATYPE = NETCDF ENSEMBLE_STAT_N_MEMBERS = 26 ENSEMBLE_STAT_ENS_THRESH = 1.0 -LOG_LEVEL = DEBUG - -LOG_ENSEMBLE_STAT_VERBOSITY = 2 ENSEMBLE_STAT_OUTPUT_PREFIX = {MODEL}_PREPBUFR_{domain}_FHR{lead?fmt=%HH} diff --git a/parm/metplus_config/stats/mesoscale/grid2obs/GenEnsProd_fcstSREF_obsPREPBUFR.conf b/parm/metplus_config/stats/mesoscale/grid2obs/GenEnsProd_fcstSREF_obsPREPBUFR.conf index 730dee879..efbff2fec 100644 --- a/parm/metplus_config/stats/mesoscale/grid2obs/GenEnsProd_fcstSREF_obsPREPBUFR.conf +++ b/parm/metplus_config/stats/mesoscale/grid2obs/GenEnsProd_fcstSREF_obsPREPBUFR.conf @@ -1,7 +1,6 @@ [config] OUTPUT_BASE = {ENV[output_base]} -LOG_LINE_FORMAT = # time looping - options are INIT, VALID, RETRO, and REALTIME @@ -22,8 +21,6 @@ METPLUS_PATH = {ENV[METPLUS_PATH]} VALID_INCREMENT = {ENV[valid_increment]} -LOOP_ORDER = times - PROCESS_LIST = GenEnsProd ENS_VAR1_NAME = TMP @@ -38,7 +35,6 @@ ENS_VAR3_LEVELS = Z10 ENS_VAR4_NAME = RH ENS_VAR4_LEVELS = Z2 - ENS_VAR5_NAME = VIS ENS_VAR5_LEVELS = L0 ENS_VAR5_OPTIONS = GRIB_lvl_typ = 1 @@ -61,7 +57,7 @@ ENS_VAR8_THRESH = lt152, lt305, lt914, lt1524, lt3048, ge914 ENS_VAR9_NAME = CAPE ENS_VAR9_LEVELS = L0 ENS_VAR9_THRESH = ge250, ge500, ge1000, ge2000 -ENS_VAR9_OPTIONS = GRIB_lvl_typ = 1 +ENS_VAR9_OPTIONS = GRIB_lvl_typ = 1; cnt_thresh = [ NA ]; cnt_logic = INTERSECTION; ENS_VAR10_NAME = PRMSL ENS_VAR10_LEVELS = L0 @@ -106,33 +102,6 @@ model = {ENV[model]} TMP_DIR = {OUTPUT_BASE}/tmp - -#GEN_ENS_PROD_NBRHD_PROB_WIDTH = 3 -#GEN_ENS_PROD_NBRHD_PROB_SHAPE = CIRCLE -#GEN_ENS_PROD_NBRHD_PROB_VLD_THRESH = 0.0 - -#GEN_ENS_PROD_NMEP_SMOOTH_VLD_THRESH = 0.0 -#GEN_ENS_PROD_NMEP_SMOOTH_SHAPE = CIRCLE -#GEN_ENS_PROD_NMEP_SMOOTH_GAUSSIAN_DX = 81.27 -#GEN_ENS_PROD_NMEP_SMOOTH_GAUSSIAN_RADIUS = 120 -#GEN_ENS_PROD_NMEP_SMOOTH_METHOD = GAUSSIAN -#GEN_ENS_PROD_NMEP_SMOOTH_WIDTH = 1 - -#GEN_ENS_PROD_INTERP_FIELD = BOTH -#GEN_ENS_PROD_INTERP_VLD_THRESH = 1.0 -#GEN_ENS_PROD_INTERP_SHAPE = SQUARE -#GEN_ENS_PROD_INTERP_TYPE_METHOD = NEAREST -#GEN_ENS_PROD_INTERP_TYPE_WIDTH = 1 - - -#GEN_ENS_PROD_OUTPUT_FLAG_ECNT = BOTH -#GEN_ENS_PROD_OUTPUT_FLAG_RPS = NONE -#GEN_ENS_PROD_OUTPUT_FLAG_RHIST = NONE -#GEN_ENS_PROD_OUTPUT_FLAG_PHIST = NONE -#GEN_ENS_PROD_OUTPUT_FLAG_ORANK = NONE -#GEN_ENS_PROD_OUTPUT_FLAG_SSVAR = NONE -#GEN_ENS_PROD_OUTPUT_FLAG_RELP = NONE - GEN_ENS_PROD_ENSEMBLE_FLAG_LATLON = TRUE GEN_ENS_PROD_ENSEMBLE_FLAG_MEAN = TRUE GEN_ENS_PROD_ENSEMBLE_FLAG_STDEV = TRUE @@ -163,9 +132,6 @@ METPLUS_CONF = {OUTPUT_BASE}/final_{model}_sfc_{ENV[vbeg]}_gen_ens_prod.conf GEN_ENS_PROD_N_MEMBERS = 26 GEN_ENS_PROD_ENS_THRESH = 1.0 -LOG_LEVEL = DEBUG -LOG_GEN_ENS_PROD_VERBOSITY = 2 - [dir] GEN_ENS_PROD_INPUT_DIR = {modelpath} GEN_ENS_PROD_OUTPUT_DIR = {OUTPUT_BASE}/stat diff --git a/parm/metplus_config/stats/mesoscale/grid2obs/Pb2nc_obsGFS_Prepbufr.conf b/parm/metplus_config/stats/mesoscale/grid2obs/Pb2nc_obsGFS_Prepbufr.conf index 7cdebfbe0..84a962608 100644 --- a/parm/metplus_config/stats/mesoscale/grid2obs/Pb2nc_obsGFS_Prepbufr.conf +++ b/parm/metplus_config/stats/mesoscale/grid2obs/Pb2nc_obsGFS_Prepbufr.conf @@ -5,8 +5,6 @@ [config] OUTPUT_BASE = {ENV[output_base]} -LOG_LINE_FORMAT = - # List of applications to run - only PB2NC for this case PROCESS_LIST = PB2NC @@ -38,21 +36,6 @@ VALID_INCREMENT = 1M # If unset, defaults to 0 (don't loop through forecast leads) LEAD_SEQ = 0 -# list of offsets in the prepBUFR input filenames to allow. List is in order of preference -# i.e. if 12, 6 is listed, it will try to use a 12 offset file and then try to use a 6 offset -# if the 12 does not exist -#PB2NC_OFFSETS = 12 - -# Order of loops to process data - Options are times, processes -# Not relevant if only one item is in the PROCESS_LIST -# times = run all wrappers in the PROCESS_LIST for a single run time, then -# increment the run time and run all wrappers again until all times have -# been evaluated. -# processes = run the first wrapper in the PROCESS_LIST for all times -# specified, then repeat for the next item in the PROCESS_LIST until all -# wrappers have been run -LOOP_ORDER = processes - # Location of MET config file to pass to PB2NC # References CONFIG_DIR from the [dir] section PB2NC_CONFIG_FILE = {METPLUS_BASE}/parm/met_config/PB2NCConfig_wrapped @@ -81,24 +64,11 @@ PB2NC_WINDOW_END = 1800 PB2NC_GRID = G212 PB2NC_POLY = PB2NC_STATION_ID = -#PB2NC_MESSAGE_TYPE = ADPUPA, ADPSFC, SFCSHP PB2NC_MESSAGE_TYPE = -#PB2NC_PB_REPORT_TYPE = 120, 220, 221, 122, 222, 223, 224, 131, 133, 233, 153, 156, 157, 180, 280, 181, 182, 281, 282, 183, 284, 187, 287 - -#PB2NC_LEVEL_RANGE_BEG = -1000 -#PB2NC_LEVEL_RANGE_END = 100000 - - -#PB2NC_LEVEL_RANGE_BEG = 1 -#PB2NC_LEVEL_RANGE_END = 255 - -#PB2NC_LEVEL_CATEGORY = 0, 1, 4, 5, 6 - PB2NC_QUALITY_MARK_THRESH = 9 # Leave empty to process all - PB2NC_OBS_BUFR_VAR_LIST = ZOB, UOB, VOB, TOB, TDO, D_CAPE, D_RH, HOVI, TOCC, CEILING, PMO #Note, there must be no space between " and ; diff --git a/parm/metplus_config/stats/mesoscale/grid2obs/PointStat_fcstSREF_obsPREPBUFR_cnv.conf b/parm/metplus_config/stats/mesoscale/grid2obs/PointStat_fcstSREF_obsPREPBUFR_cnv.conf index 3db51a734..4645a9f0a 100755 --- a/parm/metplus_config/stats/mesoscale/grid2obs/PointStat_fcstSREF_obsPREPBUFR_cnv.conf +++ b/parm/metplus_config/stats/mesoscale/grid2obs/PointStat_fcstSREF_obsPREPBUFR_cnv.conf @@ -1,8 +1,6 @@ [config] OUTPUT_BASE = {ENV[output_base]} -LOG_LINE_FORMAT = - # time looping - options are INIT, VALID, RETRO, and REALTIME LOOP_BY = VALID @@ -26,30 +24,13 @@ METPLUS_PATH = {ENV[METPLUS_PATH]} # Increment between METplus runs in seconds. Must be >= 60 # 86400 sec=24h -#VALID_INCREMENT = 21600 VALID_INCREMENT = {ENV[valid_increment]} -# Options are times, processes -# times = run all items in the PROCESS_LIST for a single initialization -# time, then repeat until all times have been evaluated. -# processes = run each item in the PROCESS_LIST for all times -# specified, then repeat for the next item in the PROCESS_LIST. -LOOP_ORDER = times # List of applications to run -#PROCESS_LIST = EnsembleStat, GridStat PROCESS_LIST = PointStat -#POINT_STAT_REGRID_TO_GRID = {ENV[regrid]} -# run pcp_combine on forecast/obs data? -#FCST_PCP_COMBINE_RUN = True -#OBS_PCP_COMBINE_RUN = True -#OBS_REGRID_DATA_PLANE_RUN = True - - # list of variables to compare - - BOTH_VAR1_NAME = HGT BOTH_VAR1_LEVELS = L0 BOTH_VAR1_THRESH = <152, <305, <914, <1524, <3048, >=914 @@ -67,7 +48,6 @@ MODEL = {ENV[MODEL]} model = {ENV[model]} domain = {ENV[domain]} ####################################################### -#POINT_STAT_REGRID_TO_GRID = ENV[regrid]} POINT_STAT_REGRID_TO_GRID = NONE OBTYPE = PREPBUFR @@ -135,13 +115,6 @@ POINT_STAT_NC_PAIRS_FLAG_GRADIENT = FALSE POINT_STAT_NC_PAIRS_FLAG_DISTANCE_MAP = FALSE POINT_STAT_NC_PAIRS_FLAG_APPLY_MASK = TRUE - -#POINT_STAT_GRID_WEIGHT_FLAG = COS_LAT - -#POINT_STAT_MET_CONFIG_OVERRIDES = grid_weight_flag = "COS_LAT"; tmp_dir = "{TMP_DIR}"; - - - ############################################################### modelhead = {ENV[modelhead]} modelpath = {ENV[modelpath]} @@ -158,9 +131,6 @@ POINT_STAT_CONFIG_FILE = {METPLUS_BASE}/parm/met_config/PointStatConfig_wrapped #Defines the name of final metplus conf METPLUS_CONF = {OUTPUT_BASE}/final_{model}_{ENV[base_model]}_{ENV[mbr]}_sfc_{ENV[vbeg]}.conf -LOG_LEVEL = DEBUG -LOG_POINT_STAT_VERBOSITY = 2 - POINT_STAT_OUTPUT_PREFIX = {MODEL}{ENV[base_model]}_{ENV[mbr]}_PREPBUFR_{domain}_FHR{lead?fmt=%HH} [dir] diff --git a/parm/metplus_config/stats/mesoscale/grid2obs/PointStat_fcstSREF_obsPREPBUFR_mean.conf b/parm/metplus_config/stats/mesoscale/grid2obs/PointStat_fcstSREF_obsPREPBUFR_mean.conf index 8c452fffe..01bfbf422 100755 --- a/parm/metplus_config/stats/mesoscale/grid2obs/PointStat_fcstSREF_obsPREPBUFR_mean.conf +++ b/parm/metplus_config/stats/mesoscale/grid2obs/PointStat_fcstSREF_obsPREPBUFR_mean.conf @@ -1,7 +1,6 @@ [config] OUTPUT_BASE = {ENV[output_base]} -LOG_LINE_FORMAT = # time looping - options are INIT, VALID, RETRO, and REALTIME @@ -21,35 +20,15 @@ VALID_END = {ENV[vday]}{ENV[vend]} METPLUS_PATH = {ENV[METPLUS_PATH]} -#conf and config path -#MET_CONFIG = {ENV[MET_CONFIG]} - # Increment between METplus runs in seconds. Must be >= 60 # 86400 sec=24h -#VALID_INCREMENT = 21600 VALID_INCREMENT = {ENV[valid_increment]} -# Options are times, processes -# times = run all items in the PROCESS_LIST for a single initialization -# time, then repeat until all times have been evaluated. -# processes = run each item in the PROCESS_LIST for all times -# specified, then repeat for the next item in the PROCESS_LIST. -LOOP_ORDER = times - # List of applications to run -#PROCESS_LIST = EnsembleStat, GridStat PROCESS_LIST = PointStat -#POINT_STAT_REGRID_TO_GRID = {ENV[regrid]} -# run pcp_combine on forecast/obs data? -#FCST_PCP_COMBINE_RUN = True -#OBS_PCP_COMBINE_RUN = True -#OBS_REGRID_DATA_PLANE_RUN = True - # list of variables to compare - - FCST_POINT_STAT_VAR1_NAME = TCDC_L0_ENS_MEAN FCST_POINT_STAT_VAR1_LEVELS = (*,*) BOTH_POINT_STAT_VAR1_THRESH = lt1.0, ge1.0, ge10, ge20, ge30, ge40, ge50, ge60, ge70, ge80, ge90, ge100 @@ -58,10 +37,11 @@ OBS_POINT_STAT_VAR1_LEVELS = L0 FCST_POINT_STAT_VAR2_NAME = CAPE_L0_ENS_MEAN FCST_POINT_STAT_VAR2_LEVELS = (*,*) +FCST_POINT_STAT_VAR2_OPTIONS = cnt_thresh = [ NA ]; cnt_logic = INTERSECTION; BOTH_POINT_STAT_VAR2_THRESH = ge250, ge500, ge1000, ge2000 OBS_POINT_STAT_VAR2_NAME = CAPE OBS_POINT_STAT_VAR2_LEVELS = L0 - +OBS_POINT_STAT_VAR2_OPTIONS = cnt_thresh = [ >0 ]; cnt_logic = INTERSECTION; FCST_POINT_STAT_VAR3_NAME = DPT_Z2_ENS_MEAN FCST_POINT_STAT_VAR3_LEVELS = (*,*) @@ -70,13 +50,9 @@ BOTH_POINT_STAT_VAR3_THRESH = ge277.59, ge283.15, ge288.71, ge294.26 OBS_POINT_STAT_VAR3_NAME = DPT OBS_POINT_STAT_VAR3_LEVELS = Z2 - FCST_IS_PROB = false FCST_GRID_STAT_PROB_THRESH = ==0.1 - -#LEAD_SEQ = 0, 6, 12, 18, 24, 30, 36, 42, 48, 54, 60 -#LEAD_SEQ = {ENV[lead_times]} lead = {ENV[lead]} LEAD_SEQ = {lead} #Other environment parameters passed from scripts @@ -84,7 +60,6 @@ MODEL = {ENV[MODEL]} model = {ENV[model]} domain = {ENV[domain]} ####################################################### -#POINT_STAT_REGRID_TO_GRID = ENV[regrid]} POINT_STAT_REGRID_TO_GRID = NONE OBTYPE = PREPBUFR @@ -147,13 +122,10 @@ POINT_STAT_NC_PAIRS_FLAG_GRADIENT = FALSE POINT_STAT_NC_PAIRS_FLAG_DISTANCE_MAP = FALSE POINT_STAT_NC_PAIRS_FLAG_APPLY_MASK = TRUE - POINT_STAT_GRID_WEIGHT_FLAG = COS_LAT POINT_STAT_MET_CONFIG_OVERRIDES = grid_weight_flag = "COS_LAT"; tmp_dir = "{TMP_DIR}"; - - ############################################################### modelhead = {ENV[modelhead]} modelpath = {ENV[modelpath]} @@ -170,9 +142,6 @@ POINT_STAT_CONFIG_FILE = {METPLUS_BASE}/parm/met_config/PointStatConfig_wrapped #Defines the name of final metplus conf METPLUS_CONF = {OUTPUT_BASE}/final_{model}_mean_sfc_{ENV[vbeg]}.conf -LOG_LEVEL = DEBUG -LOG_POINT_STAT_VERBOSITY = 2 - POINT_STAT_OUTPUT_PREFIX = {MODEL}_PREPBUFR_{domain}_mean_FHR{lead?fmt=%HH} [dir] @@ -181,7 +150,6 @@ FCST_POINT_STAT_INPUT_DIR = {OUTPUT_BASE}/stat OBS_POINT_STAT_INPUT_DIR = {obsvpath} POINT_STAT_OUTPUT_DIR = {FCST_POINT_STAT_INPUT_DIR} - [filename_templates] FCST_POINT_STAT_INPUT_TEMPLATE = GenEnsProd_{MODEL}_PREPBUFR_{domain}_FHR{lead?fmt=%HH}_{valid?fmt=%Y%m%d}_{valid?fmt=%H}0000V_ens.nc diff --git a/parm/metplus_config/stats/mesoscale/grid2obs/PointStat_fcstSREF_obsPREPBUFR_prob.conf b/parm/metplus_config/stats/mesoscale/grid2obs/PointStat_fcstSREF_obsPREPBUFR_prob.conf index f2d75e216..55318eb63 100755 --- a/parm/metplus_config/stats/mesoscale/grid2obs/PointStat_fcstSREF_obsPREPBUFR_prob.conf +++ b/parm/metplus_config/stats/mesoscale/grid2obs/PointStat_fcstSREF_obsPREPBUFR_prob.conf @@ -1,7 +1,6 @@ [config] OUTPUT_BASE = {ENV[output_base]} -LOG_LINE_FORMAT = # time looping - options are INIT, VALID, RETRO, and REALTIME LOOP_BY = VALID @@ -16,38 +15,17 @@ VALID_BEG = {ENV[vday]}{ENV[vbeg]} # End time for METplus run VALID_END = {ENV[vday]}{ENV[vend]} -#VALID_END = {now?fmt=%Y%m%d}12 - METPLUS_PATH = {ENV[METPLUS_PATH]} -#conf and config path -#MET_CONFIG = {ENV[MET_CONFIG]} - # Increment between METplus runs in seconds. Must be >= 60 # 86400 sec=24h -#VALID_INCREMENT = 21600 VALID_INCREMENT = {ENV[valid_increment]} -# Options are times, processes -# times = run all items in the PROCESS_LIST for a single initialization -# time, then repeat until all times have been evaluated. -# processes = run each item in the PROCESS_LIST for all times -# specified, then repeat for the next item in the PROCESS_LIST. -LOOP_ORDER = times - # List of applications to run #PROCESS_LIST = EnsembleStat, GridStat PROCESS_LIST = PointStat -#POINT_STAT_REGRID_TO_GRID = {ENV[regrid]} -# run pcp_combine on forecast/obs data? -#FCST_PCP_COMBINE_RUN = True -#OBS_PCP_COMBINE_RUN = True -#OBS_REGRID_DATA_PLANE_RUN = True - - # list of variables to compare - FCST_POINT_STAT_VAR1_NAME = TCDC_L0_ENS_FREQ_lt10 FCST_POINT_STAT_VAR1_LEVELS = (*,*) OBS_POINT_STAT_VAR1_NAME = TCDC @@ -169,19 +147,15 @@ OBS_POINT_STAT_VAR20_NAME = VIS OBS_POINT_STAT_VAR20_LEVELS = L0 OBS_POINT_STAT_VAR20_THRESH = ge8045 - FCST_POINT_STAT_VAR21_NAME = HGT_L0_ENS_FREQ_ge914 FCST_POINT_STAT_VAR21_LEVELS = (*,*) OBS_POINT_STAT_VAR21_NAME = HGT OBS_POINT_STAT_VAR21_LEVELS = L0 OBS_POINT_STAT_VAR21_THRESH = ge914 - FCST_IS_PROB = true FCST_GRID_STAT_PROB_THRESH = ==0.1 -#LEAD_SEQ = 0, 6, 12, 18, 24, 30, 36, 42, 48, 54, 60 -#LEAD_SEQ = {ENV[lead_times]} lead = {ENV[lead]} LEAD_SEQ = {lead} #Other environment parameters passed from scripts @@ -189,7 +163,6 @@ MODEL = {ENV[MODEL]} model = {ENV[model]} ####################################################### -#POINT_STAT_REGRID_TO_GRID = ENV[regrid]} POINT_STAT_REGRID_TO_GRID = NONE OBTYPE = PREPBUFR @@ -252,11 +225,8 @@ POINT_STAT_NC_PAIRS_FLAG_GRADIENT = FALSE POINT_STAT_NC_PAIRS_FLAG_DISTANCE_MAP = FALSE POINT_STAT_NC_PAIRS_FLAG_APPLY_MASK = TRUE - POINT_STAT_GRID_WEIGHT_FLAG = COS_LAT - - ############################################################### modelhead = {ENV[modelhead]} modelpath = {ENV[modelpath]} @@ -275,9 +245,6 @@ POINT_STAT_CONFIG_FILE = {METPLUS_BASE}/parm/met_config/PointStatConfig_wrapped #Defines the name of final metplus conf METPLUS_CONF = {OUTPUT_BASE}/final_{model}_prob_sfc_{ENV[vbeg]}.conf -LOG_LEVEL = DEBUG -LOG_POINT_STAT_VERBOSITY = 2 - POINT_STAT_OUTPUT_PREFIX = {MODEL}_PREPBUFR_{domain}_prob_FHR{lead?fmt=%HH} [dir] diff --git a/parm/metplus_config/stats/mesoscale/grid2obs/StatAnlysis_fcstSREF_obsPREPBUFR_GatherByDay.conf b/parm/metplus_config/stats/mesoscale/grid2obs/StatAnlysis_fcstSREF_obsPREPBUFR_GatherByDay.conf index 9229d7298..a0e7a7507 100755 --- a/parm/metplus_config/stats/mesoscale/grid2obs/StatAnlysis_fcstSREF_obsPREPBUFR_GatherByDay.conf +++ b/parm/metplus_config/stats/mesoscale/grid2obs/StatAnlysis_fcstSREF_obsPREPBUFR_GatherByDay.conf @@ -1,7 +1,6 @@ [config] OUTPUT_BASE = {ENV[output_base]} -LOG_LINE_FORMAT = LOOP_BY = VALID VALID_TIME_FMT = %Y%m%d%H @@ -16,16 +15,8 @@ MET_CONFIG = {ENV[MET_CONFIG]} #VERIFY: grid2grid, gridobs, precip VERIFY = {ENV[verify]} -LOOP_ORDER = times PROCESS_LIST = StatAnalysis -LOG_MET_VERBOSITY = 4 -LOG_LEVEL = DEBUG - - -#LOG_METPLUS = {LOG_DIR}/{MODEL}/{VALID_BEG}/master_metplus.statanalysis.log.{LOG_TIMESTAMP_TEMPLATE} -#FCST_VALID_HOUR_LIST = 00, 01, 02, 03, 04, 05, 06, 07, 08, 09, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23 - FCST_VALID_HOUR_LIST = GROUP_LIST_ITEMS = @@ -74,21 +65,12 @@ ALPHA_LIST = LINE_TYPE_LIST = GROUP_LIST_ITEMS = - [dir] # Full path to output stat files, can use wildcards if in multiple directories -#STAT_ANALYSIS_OUTPUT_DIR = {ENV[gather_output_dir]} STAT_ANALYSIS_OUTPUT_DIR = {OUTPUT_BASE} -#Location of input stat files and the stat file pattern to be gathered -#verify_type:ensemble_stat, grid_stat, point_stat -#MODEL1: GEFS, CMCE, ECME, NAEFS -#preci_type=BIN1, APCP24, '' - -#MODEL1_STAT_ANALYSIS_LOOKIN_DIR = {ENV[stat_file_dir]}/*{MODEL1}* MODEL1_STAT_ANALYSIS_LOOKIN_DIR = {ENV[stat_file_dir]} - [filename_templates] STAT_ANALYSIS_DUMP_ROW_TEMPLATE = {valid_beg?fmt=%Y%m%d}/{model}_grid2obs_{valid_beg?fmt=%Y%m%d}.stat STAT_ANALYSIS_OUT_STAT_TEMPLATE = diff --git a/parm/metplus_config/stats/mesoscale/precip/EnsembleStat_fcstSREF_obsCCPA_G212.conf b/parm/metplus_config/stats/mesoscale/precip/EnsembleStat_fcstSREF_obsCCPA_G212.conf index bc195c10f..04d47b971 100755 --- a/parm/metplus_config/stats/mesoscale/precip/EnsembleStat_fcstSREF_obsCCPA_G212.conf +++ b/parm/metplus_config/stats/mesoscale/precip/EnsembleStat_fcstSREF_obsCCPA_G212.conf @@ -1,7 +1,6 @@ [config] OUTPUT_BASE = {ENV[output_base]} -LOG_LINE_FORMAT = # time looping - options are INIT, VALID, RETRO, and REALTIME @@ -25,28 +24,11 @@ METPLUS_PATH = {ENV[METPLUS_PATH]} #MET_CONFIG = {ENV[MET_CONFIG]} # Increment between METplus runs in seconds. Must be >= 60 -# 86400 sec=24h -#VALID_INCREMENT = 21600 VALID_INCREMENT = {ENV[valid_increment]} -# Options are times, processes -# times = run all items in the PROCESS_LIST for a single initialization -# time, then repeat until all times have been evaluated. -# processes = run each item in the PROCESS_LIST for all times -# specified, then repeat for the next item in the PROCESS_LIST. -LOOP_ORDER = times - # List of applications to run -#PROCESS_LIST = EnsembleStat, GridStat PROCESS_LIST = EnsembleStat -#GRID_STAT_REGRID_TO_GRID = {ENV[regrid]} -# run pcp_combine on forecast/obs data? -#FCST_PCP_COMBINE_RUN = True -#OBS_PCP_COMBINE_RUN = True -#OBS_REGRID_DATA_PLANE_RUN = True - - # list of variables to compare FCST_VAR1_NAME = APCP_06 FCST_VAR1_LEVELS = A06 @@ -55,9 +37,6 @@ OBS_VAR1_NAME = {FCST_VAR1_NAME} OBS_VAR1_LEVELS = {FCST_VAR1_LEVELS} OBS_VAR1_THRESH = {FCST_VAR1_THRESH} - -#LEAD_SEQ = 0, 6, 12, 18, 24, 30, 36, 42, 48, 54, 60 -#LEAD_SEQ = {ENV[lead_times]} lead = {ENV[lead]} LEAD_SEQ = {lead} #Other environment parameters passed from scripts @@ -152,9 +131,6 @@ OBS_ENSEMBLE_STAT_INPUT_GRID_DATATYPE = GRIB ENSEMBLE_STAT_N_MEMBERS = 26 ENSEMBLE_STAT_ENS_THRESH = 0.8 -LOG_LEVEL = DEBUG - -LOG_ENSEMBLE_STAT_VERBOSITY = 2 ENSEMBLE_STAT_OUTPUT_PREFIX = {MODEL}_CCPA_{grid}_FHR{lead?fmt=%HH} [dir] diff --git a/parm/metplus_config/stats/mesoscale/precip/GenEnsProd_fcstSREF_obsCCPA_G212.conf b/parm/metplus_config/stats/mesoscale/precip/GenEnsProd_fcstSREF_obsCCPA_G212.conf index 9e504f82d..14c85f065 100644 --- a/parm/metplus_config/stats/mesoscale/precip/GenEnsProd_fcstSREF_obsCCPA_G212.conf +++ b/parm/metplus_config/stats/mesoscale/precip/GenEnsProd_fcstSREF_obsCCPA_G212.conf @@ -1,8 +1,6 @@ [config] OUTPUT_BASE = {ENV[output_base]} -LOG_LINE_FORMAT = - # time looping - options are INIT, VALID, RETRO, and REALTIME LOOP_BY = VALID @@ -22,8 +20,6 @@ METPLUS_PATH = {ENV[METPLUS_PATH]} VALID_INCREMENT = {ENV[valid_increment]} -LOOP_ORDER = times - PROCESS_LIST = GenEnsProd ENS_VAR1_NAME = APCP_06 @@ -42,33 +38,6 @@ grid = {ENV[grid]} TMP_DIR = {OUTPUT_BASE}/tmp - -#GEN_ENS_PROD_NBRHD_PROB_WIDTH = 3 -#GEN_ENS_PROD_NBRHD_PROB_SHAPE = CIRCLE -#GEN_ENS_PROD_NBRHD_PROB_VLD_THRESH = 0.0 - -#GEN_ENS_PROD_NMEP_SMOOTH_VLD_THRESH = 0.0 -#GEN_ENS_PROD_NMEP_SMOOTH_SHAPE = CIRCLE -#GEN_ENS_PROD_NMEP_SMOOTH_GAUSSIAN_DX = 81.27 -#GEN_ENS_PROD_NMEP_SMOOTH_GAUSSIAN_RADIUS = 120 -#GEN_ENS_PROD_NMEP_SMOOTH_METHOD = GAUSSIAN -#GEN_ENS_PROD_NMEP_SMOOTH_WIDTH = 1 - -#GEN_ENS_PROD_INTERP_FIELD = BOTH -#GEN_ENS_PROD_INTERP_VLD_THRESH = 1.0 -#GEN_ENS_PROD_INTERP_SHAPE = SQUARE -#GEN_ENS_PROD_INTERP_TYPE_METHOD = NEAREST -#GEN_ENS_PROD_INTERP_TYPE_WIDTH = 1 - - -#GEN_ENS_PROD_OUTPUT_FLAG_ECNT = BOTH -#GEN_ENS_PROD_OUTPUT_FLAG_RPS = NONE -#GEN_ENS_PROD_OUTPUT_FLAG_RHIST = NONE -#GEN_ENS_PROD_OUTPUT_FLAG_PHIST = NONE -#GEN_ENS_PROD_OUTPUT_FLAG_ORANK = NONE -#GEN_ENS_PROD_OUTPUT_FLAG_SSVAR = NONE -#GEN_ENS_PROD_OUTPUT_FLAG_RELP = NONE - GEN_ENS_PROD_ENSEMBLE_FLAG_LATLON = TRUE GEN_ENS_PROD_ENSEMBLE_FLAG_MEAN = TRUE GEN_ENS_PROD_ENSEMBLE_FLAG_STDEV = TRUE diff --git a/parm/metplus_config/stats/mesoscale/precip/GridStat_fcstSREF_obsCCPA_mean_G212.conf b/parm/metplus_config/stats/mesoscale/precip/GridStat_fcstSREF_obsCCPA_mean_G212.conf index 214b99ce3..5df65450b 100755 --- a/parm/metplus_config/stats/mesoscale/precip/GridStat_fcstSREF_obsCCPA_mean_G212.conf +++ b/parm/metplus_config/stats/mesoscale/precip/GridStat_fcstSREF_obsCCPA_mean_G212.conf @@ -1,7 +1,6 @@ [config] OUTPUT_BASE = {ENV[output_base]} -LOG_LINE_FORMAT = # time looping - options are INIT, VALID, RETRO, and REALTIME LOOP_BY = VALID @@ -11,43 +10,20 @@ VALID_TIME_FMT = %Y%m%d%H # Start time for METplus run VALID_BEG = {ENV[vday]}{ENV[vbeg]} -# # End time for METplus run VALID_END = {ENV[vday]}{ENV[vend]} -#VALID_END = {now?fmt=%Y%m%d}12 METPLUS_PATH = {ENV[METPLUS_PATH]} -#conf and config path -#MET_CONFIG = {ENV[MET_CONFIG]} - # Increment between METplus runs in seconds. Must be >= 60 -# 86400 sec=24h -#VALID_INCREMENT = 21600 VALID_INCREMENT = {ENV[valid_increment]} -# Options are times, processes -# times = run all items in the PROCESS_LIST for a single initialization -# time, then repeat until all times have been evaluated. -# processes = run each item in the PROCESS_LIST for all times -# specified, then repeat for the next item in the PROCESS_LIST. -LOOP_ORDER = times - # List of applications to run PROCESS_LIST = GridStat -#PROCESS_LIST = EnsembleStat - -#GRID_STAT_REGRID_TO_GRID = {ENV[regrid]} -# run pcp_combine on forecast/obs data? -#FCST_PCP_COMBINE_RUN = True -#OBS_PCP_COMBINE_RUN = True -#OBS_REGRID_DATA_PLANE_RUN = True -# list of variables to compare - FCST_GRID_STAT_VAR1_NAME = APCP_06_A6_ENS_MEAN FCST_GRID_STAT_VAR1_LEVELS = (*,*) BOTH_GRID_STAT_VAR1_THRESH = ge0.01, ge0.1, ge0.5, ge1, ge5, ge10, ge25, ge50, ge75 @@ -58,8 +34,6 @@ OBS_GRID_STAT_VAR1_LEVELS = A6 FCST_IS_PROB = false FCST_GRID_STAT_PROB_THRESH = ==0.1 -#LEAD_SEQ = 0, 6, 12, 18, 24, 30, 36, 42, 48, 54, 60 -#LEAD_SEQ = {ENV[lead_times]} lead = {ENV[lead]} LEAD_SEQ = {lead} #Other environment parameters passed from scripts @@ -146,8 +120,6 @@ obsvhead = {ENV[obsvhead]} obsvgrid = {ENV[obsvgrid]} extradir = {ENV[extradir]} - - # location of ensemble_stat and grid_stat MET config files GRID_STAT_CONFIG_FILE = {METPLUS_BASE}/parm/met_config/GridStatConfig_wrapped @@ -156,9 +128,6 @@ METPLUS_CONF = {OUTPUT_BASE}/final_grid_Stat_{model}_ccpa_mean_G212.conf OBS_GRID_STAT_INPUT_GRID_DATATYPE = GRIB -LOG_LEVEL = DEBUG - -LOG_GRID_STAT_VERBOSITY = 2 GRID_STAT_OUTPUT_PREFIX = {MODEL}_CCPA_{grid}_MEAN [dir] @@ -166,12 +135,8 @@ FCST_GRID_STAT_INPUT_DIR = {OUTPUT_BASE}/stat OBS_GRID_STAT_INPUT_DIR = {obsvpath} GRID_STAT_OUTPUT_DIR = {FCST_GRID_STAT_INPUT_DIR} - - - [filename_templates] - FCST_GRID_STAT_INPUT_TEMPLATE = GenEnsProd_{MODEL}_CCPA_{grid}_FHR{lead?fmt=%HH}_{valid?fmt=%Y%m%d}_{valid?fmt=%H}0000V_ens.nc OBS_GRID_STAT_INPUT_TEMPLATE = {obsvhead}.{valid?fmt=%Y%m%d}/{obsvhead}.t{valid?fmt=%H}z.{obsvgrid}.06h.f00.nc diff --git a/parm/metplus_config/stats/mesoscale/precip/GridStat_fcstSREF_obsCCPA_mean_G240.conf b/parm/metplus_config/stats/mesoscale/precip/GridStat_fcstSREF_obsCCPA_mean_G240.conf index 6f829b358..9a674e93b 100755 --- a/parm/metplus_config/stats/mesoscale/precip/GridStat_fcstSREF_obsCCPA_mean_G240.conf +++ b/parm/metplus_config/stats/mesoscale/precip/GridStat_fcstSREF_obsCCPA_mean_G240.conf @@ -1,7 +1,6 @@ [config] OUTPUT_BASE = {ENV[output_base]} -LOG_LINE_FORMAT = # time looping - options are INIT, VALID, RETRO, and REALTIME LOOP_BY = VALID @@ -11,42 +10,23 @@ VALID_TIME_FMT = %Y%m%d%H # Start time for METplus run VALID_BEG = {ENV[vday]}{ENV[vbeg]} -# # End time for METplus run VALID_END = {ENV[vday]}{ENV[vend]} -#VALID_END = {now?fmt=%Y%m%d}12 METPLUS_PATH = {ENV[METPLUS_PATH]} -#conf and config path -#MET_CONFIG = {ENV[MET_CONFIG]} - # Increment between METplus runs in seconds. Must be >= 60 -# 86400 sec=24h -#VALID_INCREMENT = 21600 VALID_INCREMENT = {ENV[valid_increment]} -# Options are times, processes -# times = run all items in the PROCESS_LIST for a single initialization -# time, then repeat until all times have been evaluated. -# processes = run each item in the PROCESS_LIST for all times -# specified, then repeat for the next item in the PROCESS_LIST. -LOOP_ORDER = times - # List of applications to run PROCESS_LIST = GridStat #PROCESS_LIST = EnsembleStat GRID_STAT_REGRID_TO_GRID = {ENV[regrid]} -#FCST_PCP_COMBINE_RUN = True -#OBS_PCP_COMBINE_RUN = True -#OBS_REGRID_DATA_PLANE_RUN = True - # list of variables to compare - FCST_GRID_STAT_VAR1_NAME = APCP_06_A6_ENS_MEAN FCST_GRID_STAT_VAR1_LEVELS = (*,*) BOTH_GRID_STAT_VAR1_THRESH = ge0.01, ge0.1, ge0.5, ge1, ge5, ge10, ge25, ge50, ge75 @@ -145,8 +125,6 @@ obsvhead = {ENV[obsvhead]} obsvgrid = {ENV[obsvgrid]} extradir = {ENV[extradir]} - - # location of ensemble_stat and grid_stat MET config files GRID_STAT_CONFIG_FILE = {METPLUS_BASE}/parm/met_config/GridStatConfig_wrapped @@ -165,12 +143,8 @@ FCST_GRID_STAT_INPUT_DIR = {OUTPUT_BASE}/stat OBS_GRID_STAT_INPUT_DIR = {obsvpath} GRID_STAT_OUTPUT_DIR = {FCST_GRID_STAT_INPUT_DIR} - - - [filename_templates] - FCST_GRID_STAT_INPUT_TEMPLATE = GenEnsProd_{MODEL}_CCPA_G212_FHR{lead?fmt=%HH}_{valid?fmt=%Y%m%d}_{valid?fmt=%H}0000V_ens.nc OBS_GRID_STAT_INPUT_TEMPLATE = {obsvhead}.{valid?fmt=%Y%m%d}/{obsvhead}.t{valid?fmt=%H}z.{obsvgrid}.06h.f00.nc diff --git a/parm/metplus_config/stats/mesoscale/precip/GridStat_fcstSREF_obsCCPA_prob_G212.conf b/parm/metplus_config/stats/mesoscale/precip/GridStat_fcstSREF_obsCCPA_prob_G212.conf index 8c45725dd..612560b60 100755 --- a/parm/metplus_config/stats/mesoscale/precip/GridStat_fcstSREF_obsCCPA_prob_G212.conf +++ b/parm/metplus_config/stats/mesoscale/precip/GridStat_fcstSREF_obsCCPA_prob_G212.conf @@ -1,8 +1,6 @@ [config] OUTPUT_BASE = {ENV[output_base]} -LOG_LINE_FORMAT = - # time looping - options are INIT, VALID, RETRO, and REALTIME LOOP_BY = VALID @@ -12,42 +10,17 @@ VALID_TIME_FMT = %Y%m%d%H # Start time for METplus run VALID_BEG = {ENV[vday]}{ENV[vbeg]} -# # End time for METplus run VALID_END = {ENV[vday]}{ENV[vend]} -#VALID_END = {now?fmt=%Y%m%d}12 - METPLUS_PATH = {ENV[METPLUS_PATH]} -#conf and config path -#MET_CONFIG = {ENV[MET_CONFIG]} - # Increment between METplus runs in seconds. Must be >= 60 -# 86400 sec=24h -#VALID_INCREMENT = 21600 VALID_INCREMENT = {ENV[valid_increment]} -# Options are times, processes -# times = run all items in the PROCESS_LIST for a single initialization -# time, then repeat until all times have been evaluated. -# processes = run each item in the PROCESS_LIST for all times -# specified, then repeat for the next item in the PROCESS_LIST. -LOOP_ORDER = times - # List of applications to run PROCESS_LIST = GridStat -#PROCESS_LIST = EnsembleStat - -#GRID_STAT_REGRID_TO_GRID = {ENV[regrid]} -# run pcp_combine on forecast/obs data? -#FCST_PCP_COMBINE_RUN = True -#OBS_PCP_COMBINE_RUN = True -#OBS_REGRID_DATA_PLANE_RUN = True - - -# list of variables to compare FCST_GRID_STAT_VAR1_NAME = APCP_06_A6_ENS_FREQ_gt0.01 FCST_GRID_STAT_VAR1_LEVELS = (*,*) @@ -107,8 +80,6 @@ OBS_GRID_STAT_VAR9_THRESH = gt75 FCST_IS_PROB = true FCST_GRID_STAT_PROB_THRESH = ==0.1 -#LEAD_SEQ = 0, 6, 12, 18, 24, 30, 36, 42, 48, 54, 60 -#LEAD_SEQ = {ENV[lead_times]} lead = {ENV[lead]} LEAD_SEQ = {lead} #Other environment parameters passed from scripts @@ -130,14 +101,6 @@ GRID_STAT_REGRID_WIDTH = 2 GRID_STAT_REGRID_VLD_THRESH = 0.5 GRID_STAT_REGRID_SHAPE = SQUARE -#GRID_STAT_CLIMO_MEAN_FIELD = {ENSEMBLE_STAT_CLIMO_MEAN_FIELD} -#GRID_STAT_CLIMO_MEAN_TIME_INTERP_METHOD = DW_MEAN -#GRID_STAT_CLIMO_MEAN_MATCH_MONTH = FALSE -#GRID_STAT_CLIMO_MEAN_DAY_INTERVAL = 31 -#GRID_STAT_CLIMO_MEAN_HOUR_INTERVAL = 18 - -#ENSEMBLE_STAT_MASK_GRID = [] - GRID_STAT_MASK_GRID = GRID_STAT_MASK_POLY = {MASKPATH}/Bukovsky_G212_CONUS.nc @@ -217,9 +180,6 @@ METPLUS_CONF = {OUTPUT_BASE}/final_grid_Stat_{model}_ccpa_prob.conf OBS_GRID_STAT_INPUT_GRID_DATATYPE = GRIB -LOG_LEVEL = DEBUG - -LOG_GRID_STAT_VERBOSITY = 2 GRID_STAT_OUTPUT_PREFIX = {MODEL}_CCPA_{grid}_PROB [dir] @@ -228,11 +188,6 @@ FCST_GRID_STAT_INPUT_DIR = {OUTPUT_BASE}/stat OBS_GRID_STAT_INPUT_DIR = {obsvpath} GRID_STAT_OUTPUT_DIR = {FCST_GRID_STAT_INPUT_DIR} - -#GRID_STAT_CLIMO_MEAN_INPUT_DIR = {climpath_apcp24_prob} - - - [filename_templates] FCST_GRID_STAT_INPUT_TEMPLATE = GenEnsProd_{MODEL}_CCPA_{grid}_FHR{lead?fmt=%HH}_{valid?fmt=%Y%m%d}_{valid?fmt=%H}0000V_ens.nc diff --git a/parm/metplus_config/stats/mesoscale/precip/PcpCombine_fcstSREF_APCP06h.conf b/parm/metplus_config/stats/mesoscale/precip/PcpCombine_fcstSREF_APCP06h.conf index 5fd7306c9..4848daf6e 100644 --- a/parm/metplus_config/stats/mesoscale/precip/PcpCombine_fcstSREF_APCP06h.conf +++ b/parm/metplus_config/stats/mesoscale/precip/PcpCombine_fcstSREF_APCP06h.conf @@ -1,7 +1,6 @@ [config] OUTPUT_BASE = {ENV[output_base]} -LOG_LINE_FORMAT = PROCESS_LIST = PCPCombine @@ -13,8 +12,6 @@ VALID_INCREMENT = 1H LEAD_SEQ = {ENV[fhr]} -LOOP_ORDER = times - FCST_PCP_COMBINE_CONSTANT_INIT = True FCST_PCP_COMBINE_RUN = True @@ -26,8 +23,6 @@ FCST_PCP_COMBINE_INPUT_TEMPLATE = sref_{ENV[base]}.t{ENV[fvhr]}z.pgrb212.{ENV[mb FCST_PCP_COMBINE_OUTPUT_DIR = {OUTPUT_BASE} FCST_PCP_COMBINE_OUTPUT_TEMPLATE = sref_{ENV[base]}.t{ENV[fvhr]}z.{ENV[mb]}.pgrb212.6hr.f{ENV[fhr]}.nc -#LOG_PCP_COMBINE_VERBOSITY = 2 - FCST_IS_PROB = false FCST_PCP_COMBINE_INPUT_DATATYPE = GRIB diff --git a/parm/metplus_config/stats/mesoscale/precip/PcpCombine_fcstSREF_APCP24h.conf b/parm/metplus_config/stats/mesoscale/precip/PcpCombine_fcstSREF_APCP24h.conf index ac5869f14..ca8b6622b 100644 --- a/parm/metplus_config/stats/mesoscale/precip/PcpCombine_fcstSREF_APCP24h.conf +++ b/parm/metplus_config/stats/mesoscale/precip/PcpCombine_fcstSREF_APCP24h.conf @@ -1,7 +1,6 @@ [config] OUTPUT_BASE = {ENV[output_base]} -LOG_LINE_FORMAT = PROCESS_LIST = PCPCombine @@ -13,8 +12,6 @@ INIT_INCREMENT = 1H LEAD_SEQ = {ENV[lead]} -LOOP_ORDER = times - FCST_PCP_COMBINE_CONSTANT_INIT = True FCST_PCP_COMBINE_RUN = True @@ -26,8 +23,6 @@ FCST_PCP_COMBINE_INPUT_TEMPLATE = sref.t{init?fmt=%H}z.pgrb212.mean.fhr{lead?fmt FCST_PCP_COMBINE_OUTPUT_DIR = {OUTPUT_BASE} FCST_PCP_COMBINE_OUTPUT_TEMPLATE = sref.t{init?fmt=%H}z.pgrb212.24mean.f{lead?fmt=%H}.nc -LOG_PCP_COMBINE_VERBOSITY = 5 - FCST_IS_PROB = false FCST_PCP_COMBINE_INPUT_DATATYPE = GRIB diff --git a/parm/metplus_config/stats/mesoscale/precip/StatAnlysis_fcstSREF_obsCCPA_GatherByDay.conf b/parm/metplus_config/stats/mesoscale/precip/StatAnlysis_fcstSREF_obsCCPA_GatherByDay.conf index 9b7cf1c54..e9358391f 100755 --- a/parm/metplus_config/stats/mesoscale/precip/StatAnlysis_fcstSREF_obsCCPA_GatherByDay.conf +++ b/parm/metplus_config/stats/mesoscale/precip/StatAnlysis_fcstSREF_obsCCPA_GatherByDay.conf @@ -1,7 +1,6 @@ [config] OUTPUT_BASE = {ENV[output_base]} -LOG_LINE_FORMAT = LOOP_BY = VALID VALID_TIME_FMT = %Y%m%d%H @@ -13,19 +12,13 @@ VALID_INCREMENT = {ENV[valid_increment]} MET_CONFIG = {ENV[MET_CONFIG]} -#VERIFY: grid2grid, gridobs, precip VERIFY = {ENV[verify]} -LOOP_ORDER = times PROCESS_LIST = StatAnalysis LOG_MET_VERBOSITY = 4 LOG_LEVEL = DEBUG - -#LOG_METPLUS = {LOG_DIR}/{MODEL}/{VALID_BEG}/master_metplus.statanalysis.log.{LOG_TIMESTAMP_TEMPLATE} -#FCST_VALID_HOUR_LIST = 00, 01, 02, 03, 04, 05, 06, 07, 08, 09, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23 - FCST_VALID_HOUR_LIST = GROUP_LIST_ITEMS = @@ -45,10 +38,6 @@ STAT_ANALYSIS_CONFIG_FILE = {METPLUS_BASE}/parm/met_config/STATAnalysisConfig_wr # stat_analysis job info STAT_ANALYSIS_JOB_NAME = filter -# if using -dump_row, put in JOBS_ARGS "-dump_row [dump_row_filename]" -# if using -out_stat, put in JOBS_ARGS "-out_stat [out_stat_filename]" -# METplus will fill in filename - STAT_ANALYSIS_JOB_ARGS = -dump_row [dump_row_filename] DESC_LIST = @@ -77,18 +66,11 @@ GROUP_LIST_ITEMS = [dir] # Full path to output stat files, can use wildcards if in multiple directories -#STAT_ANALYSIS_OUTPUT_DIR = {ENV[gather_output_dir]} STAT_ANALYSIS_OUTPUT_DIR = {OUTPUT_BASE} -#Location of input stat files and the stat file pattern to be gathered -#verify_type:ensemble_stat, grid_stat, point_stat -#MODEL1: GEFS, CMCE, ECME, NAEFS -#preci_type=BIN1, APCP24, '' - #MODEL1_STAT_ANALYSIS_LOOKIN_DIR = {ENV[stat_file_dir]}/*{MODEL1}* MODEL1_STAT_ANALYSIS_LOOKIN_DIR = {ENV[stat_file_dir]} - [filename_templates] STAT_ANALYSIS_DUMP_ROW_TEMPLATE = {valid_beg?fmt=%Y%m%d}/{model}_precip_{valid_beg?fmt=%Y%m%d}.stat STAT_ANALYSIS_OUT_STAT_TEMPLATE = diff --git a/scripts/plots/mesoscale/exevs_mesoscale_sref_cape_plots.sh b/scripts/plots/mesoscale/exevs_mesoscale_sref_cape_plots.sh index 2001fc1e4..97d9f2a3a 100755 --- a/scripts/plots/mesoscale/exevs_mesoscale_sref_cape_plots.sh +++ b/scripts/plots/mesoscale/exevs_mesoscale_sref_cape_plots.sh @@ -9,18 +9,13 @@ set -x export PYTHONPATH=$HOMEevs/ush/$COMPONENT:$PYTHONPATH -cd $DATA +mkdir -p $DATA/scripts +cd $DATA/scripts -export prune_dir=$DATA/data -export save_dir=$DATA/out export output_base_dir=$DATA/stat_archive -export log_metplus=$DATA/logs/GENS_verif_plotting_job.out -mkdir -p $prune_dir -mkdir -p $save_dir mkdir -p $output_base_dir -mkdir -p $DATA/logs -restart=$COMOUTplots/restart/$past_days/sref_cape_plots +restart=$COMOUTplots/restart/$last_days/sref_cape_plots if [ ! -d $restart ] ; then mkdir -p $restart fi @@ -37,7 +32,7 @@ model_list='GEFS SREF' models='GEFS, SREF' n=0 -while [ $n -le $past_days ] ; do +while [ $n -le $last_days ] ; do hrs=$((n*24)) first_day=`$NDATE -$hrs ${VDATE}00|cut -c1-8` n=$((n+1)) @@ -47,11 +42,10 @@ export init_beg=$first_day export valid_beg=$first_day #************************************************************************* -# Virtual link the sref's stat data files of past 90 days +# Virtual link the sref's stat data files of last 90 days #************************************************************************** n=0 -while [ $n -le $past_days ] ; do - #hrs=`expr $n \* 24` +while [ $n -le $last_days ] ; do hrs=$((n*24)) day=`$NDATE -$hrs ${VDATE}00|cut -c1-8` echo $day @@ -63,15 +57,8 @@ done VX_MASK_LIST="CONUS" export fcst_init_hour="0,3,6,9,12,15,18,21" -#export fcst_valid_hour="0,6,12,18" init_time='init00_to_21z' -export plot_dir=$DATA/out/sfc_upper/${valid_beg}-${valid_end} -#For restart: -if [ ! -d $plot_dir ] ; then - mkdir -p $plot_dir -fi - verif_case=grid2obs line_type='ctc' score_types='lead_average threshold_average' @@ -91,12 +78,10 @@ for stat in ets fbias; do valid_times="00 06 12 18" thresholds="250 500 1000 2000" export fcst_group=one_group - #export fcst_lead=" 6, 9, 12, 15, 18, 21, 24, 27, 30, 33, 36, 39, 42, 45, 48, 51, 54, 57, 60, 63, 66, 69, 72, 75, 78, 81, 84, 87 elif [ $score_type = threshold_average ] ; then valid_times="00 06 12 18" thresholds=">=250,>=500,>=1000,>=2000" export fcst_group="group1 group2 group3 group4 group5 group6" - #export fcst_lead=" 6,9,12,15,18,21,24,27 30, 33, 36, 39, 42, 45, 48, 51, 54, 57, 60, 63, 66, 69, 72, 75, 78, 81, 84, 87 fi for valid_time in $valid_times ; do @@ -107,16 +92,22 @@ for stat in ets fbias; do fcst_lead=" 6, 9, 12, 15, 18, 21, 24, 27, 30, 33, 36, 39, 42, 45, 48, 51, 54, 57, 60, 63, 66, 69, 72, 75, 78, 81, 84, 87" elif [ $group = group1 ] ; then fcst_lead=" 6, 9, 12, 15, 18, 21, 24, 27" + lead="f6-9-12-15-18-21-24-27" elif [ $group = group2 ] ; then fcst_lead="12, 15, 18, 21, 24, 27, 30, 33, 36, 39" + lead="f12_to_f39" elif [ $group = group3 ] ; then fcst_lead="24, 27, 30, 33, 36, 39, 42, 45, 48, 51" + lead="f24_to_f51" elif [ $group = group4 ] ; then fcst_lead="36, 39, 42, 45, 48, 51, 54, 57, 60, 63" + lead="f36_to_f63" elif [ $group = group5 ] ; then fcst_lead="48, 51, 54, 57, 60, 63, 66, 69, 72, 75" + lead="f48_to_f75" elif [ $group = group6 ] ; then fcst_lead="60, 63, 66, 69, 72, 75, 78, 81, 84, 87" + lead="f60_to_f87" fi var=`echo $VAR | tr '[A-Z]' '[a-z]'` @@ -147,6 +138,16 @@ for stat in ets fbias; do verif_type=conus_sfc + save_dir=$DATA/plots/run_${stat}.${score_type}.${valid_time}.${group}.${thresh} + plot_dir=$save_dir/sfc_upper/${valid_beg}-${valid_end} + mkdir -p $plot_dir + mkdir -p $save_dir/data + + echo "#!/bin/ksh" >> run_${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh + echo "export save_dir=$save_dir" >> run_${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh + echo "export log_metplus=$save_dir/log_verif_plotting_job.out" >> run_${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh + echo "export prune_dir=$save_dir/data" >> run_${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh + echo "export PLOT_TYPE=$score_type" >> run_${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh echo "export field=${var}_${level}" >> run_${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh @@ -190,26 +191,18 @@ for stat in ets fbias; do chmod +x run_py.${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh - echo "${DATA}/run_py.${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh" >> run_${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh + echo "${DATA}/scripts/run_py.${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh" >> run_${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh - #Save for restart: + #Tar files and save for restart: echo "if [ ${score_type} = lead_average ] ; then " >> run_${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh - echo " cp ${plot_dir}/${score_type}_regional_conus_valid_${valid_time}z_cape_${stat}_*${thresh}.png $restart/." >> run_${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh + echo " if [ -s $plot_dir/${score_type}_regional_conus_valid_${valid_time}z_cape_${stat}_*${thresh}.png ] ; then cp ${plot_dir}/${score_type}_regional_conus_valid_${valid_time}z_cape_${stat}_*${thresh}.png $restart ; >$restart/run_${stat}.${score_type}.${valid_time}.${group}.${thresh}.completed ; fi " >> run_${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh echo "else" >> run_${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh - echo " cp ${plot_dir}/${score_type}_regional_conus_valid_${valid_time}z_cape_${stat}_*.png $restart/." >> run_${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh + echo " if [ -s ${plot_dir}/${score_type}_regional_conus_valid_${valid_time}z_cape_${stat}_${lead}.png ] ; then cp ${plot_dir}/${score_type}_regional_conus_valid_${valid_time}z_cape_${stat}_${lead}.png $restart ; >$restart/run_${stat}.${score_type}.${valid_time}.${group}.${thresh}.completed ; fi " >> run_${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh echo "fi " >> run_${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh - echo "[[ $? = 0 ]] && >$restart/run_${stat}.${score_type}.${valid_time}.${group}.${thresh}.completed" >> run_${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh chmod +x run_${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh - echo "${DATA}/run_${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh" >> run_all_poe.sh + echo "${DATA}/scripts/run_${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh" >> run_all_poe.sh - else - #For restart - if [ ${score_type} = lead_average ] ; then - cp $restart/${score_type}_regional_conus_valid_${valid_time}z_cape_${stat}_*${thresh}.png ${plot_dir}/. - else - cp $restart/${score_type}_regional_conus_valid_${valid_time}z_cape_${stat}_*.png ${plot_dir}/. - fi fi done # enf of threshold @@ -229,10 +222,10 @@ chmod +x run_all_poe.sh # ************************************************************************** if [ $run_mpi = yes ] ; then - mpiexec -np 80 -ppn 80 --cpu-bind verbose,depth cfp ${DATA}/run_all_poe.sh + mpiexec -np 80 -ppn 80 --cpu-bind verbose,depth cfp ${DATA}/scripts/run_all_poe.sh export err=$?; err_chk else - ${DATA}/run_all_poe.sh + ${DATA}/scripts/run_all_poe.sh export err=$?; err_chk fi @@ -240,7 +233,7 @@ fi #************************************************** # Change plot file names to meet the EVS standard #************************************************** -cd $plot_dir +cd $restart var=cape levels=L0 @@ -277,11 +270,11 @@ for stat in $stats ; do if [ $score_type = lead_average ] ; then if [ -s ${score_type}_regional_conus_valid_${valid}_${var}_${stat}_${threshold}.png ] ; then - mv ${score_type}_regional_conus_valid_${valid}_${var}_${stat}_${threshold}.png evs.sref.${stat}.${var_level}_${threshold}.last${past_days}days.${scoretype}.valid_${valid}.buk_conus.png + mv ${score_type}_regional_conus_valid_${valid}_${var}_${stat}_${threshold}.png evs.sref.${stat}.${var_level}_${threshold}.last${last_days}days.${scoretype}_valid${valid}.buk_conus.png fi elif [ $score_type = threshold_average ] ; then if [ -s ${score_type}_regional_conus_valid_${valid}_${var}_${stat}_${lead}.png ] ; then - mv ${score_type}_regional_conus_valid_${valid}_${var}_${stat}_${lead}.png evs.sref.${stat}.${var_level}.last${past_days}days.${scoretype}.valid_${valid}.${new_lead}.buk_conus.png + mv ${score_type}_regional_conus_valid_${valid}_${var}_${stat}_${lead}.png evs.sref.${stat}.${var_level}.last${last_days}days.${scoretype}_valid${valid}_${new_lead}.buk_conus.png fi fi @@ -291,16 +284,28 @@ for stat in $stats ; do done done +if [ -s evs*.png ] ; then + tar -cvf evs.plots.sref.cape.last${last_days}days.v${VDATE}.tar evs*.png +fi -tar -cvf evs.plots.sref.cape.past${past_days}days.v${VDATE}.tar *.png +# Cat the plotting log files +log_dir="$DATA/plots" +if [ -s $log_dir/*/log*.out ]; then + log_files=`ls $log_dir/*/log*.out` + for log_file in $log_files ; do + echo "Start: $log_file" + cat "$log_file" + echo "End: $log_file" + done +fi -if [ $SENDCOM = YES ] && [ -s evs.plots.sref.cape.past${past_days}days.v${VDATE}.tar ] ; then - cp -v evs.plots.sref.cape.past${past_days}days.v${VDATE}.tar $COMOUTplots/. +if [ $SENDCOM = YES ] && [ -s evs.plots.sref.cape.last${last_days}days.v${VDATE}.tar ] ; then + cp -v evs.plots.sref.cape.last${last_days}days.v${VDATE}.tar $COMOUTplots/. fi if [ $SENDDBN = YES ] ; then - $DBNROOT/bin/dbn_alert MODEL EVS_RZDM $job $COMOUTplots/evs.plots.sref.cape.past${past_days}days.v${VDATE}.tar + $DBNROOT/bin/dbn_alert MODEL EVS_RZDM $job $COMOUTplots/evs.plots.sref.cape.last${last_days}days.v${VDATE}.tar fi diff --git a/scripts/plots/mesoscale/exevs_mesoscale_sref_cloud_plots.sh b/scripts/plots/mesoscale/exevs_mesoscale_sref_cloud_plots.sh index 2b76a7b52..5e79eec76 100755 --- a/scripts/plots/mesoscale/exevs_mesoscale_sref_cloud_plots.sh +++ b/scripts/plots/mesoscale/exevs_mesoscale_sref_cloud_plots.sh @@ -8,18 +8,13 @@ set -x export PYTHONPATH=$HOMEevs/ush/$COMPONENT:$PYTHONPATH -cd $DATA +mkdir -p $DATA/scripts +cd $DATA/scripts -export prune_dir=$DATA/data -export save_dir=$DATA/out export output_base_dir=$DATA/stat_archive -export log_metplus=$DATA/logs/GENS_verif_plotting_job.out -mkdir -p $prune_dir -mkdir -p $save_dir mkdir -p $output_base_dir -mkdir -p $DATA/logs -restart=$COMOUTplots/restart/$past_days/sref_cloud_plots +restart=$COMOUTplots/restart/$last_days/sref_cloud_plots if [ ! -d $restart ] ; then mkdir -p $restart fi @@ -36,10 +31,10 @@ model_list='GEFS SREF' models='GEFS, SREF' #************************************************************************* -## Virtual link the sref's stat data files of past 90 days +## Virtual link the sref's stat data files of last 90 days ##************************************************************************** n=0 -while [ $n -le $past_days ] ; do +while [ $n -le $last_days ] ; do hrs=$((n*24)) first_day=`$NDATE -$hrs ${VDATE}00|cut -c1-8` n=$((n+1)) @@ -49,11 +44,12 @@ export init_beg=$first_day export valid_beg=$first_day n=0 -while [ $n -le $past_days ] ; do +while [ $n -le $last_days ] ; do hrs=$((n*24)) day=`$NDATE -$hrs ${VDATE}00|cut -c1-8` echo $day $USHevs/mesoscale/evs_get_sref_stat_file_link_plots.sh $day "$model_list" + export err=$?; err_chk n=$((n+1)) done @@ -61,14 +57,8 @@ done VX_MASK_LIST="CONUS" export fcst_init_hour="0,3,6,9,12,15,18,21" -#export fcst_valid_hour="0,6,12,18" init_time='init00_to_21z' -export plot_dir=$DATA/out/sfc_upper/${valid_beg}-${valid_end} -if [ ! -d $plot_dir ] ; then - mkdir -p $plot_dir -fi - verif_case=grid2obs line_type='ctc' score_types='lead_average threshold_average' @@ -103,16 +93,22 @@ for stat in ets fbias; do fcst_lead=" 6, 9, 12, 15, 18, 21, 24, 27, 30, 33, 36, 39, 42, 45, 48, 51, 54, 57, 60, 63, 66, 69, 72, 75, 78, 81, 84, 87" elif [ $group = group1 ] ; then fcst_lead=" 6, 9, 12, 15, 18, 21, 24, 27" + lead="f6-9-12-15-18-21-24-27" elif [ $group = group2 ] ; then fcst_lead="12, 15, 18, 21, 24, 27, 30, 33, 36, 39" + lead="f12_to_f39" elif [ $group = group3 ] ; then fcst_lead="24, 27, 30, 33, 36, 39, 42, 45, 48, 51" + lead="f24_to_f51" elif [ $group = group4 ] ; then fcst_lead="36, 39, 42, 45, 48, 51, 54, 57, 60, 63" + lead="f36_to_f63" elif [ $group = group5 ] ; then fcst_lead="48, 51, 54, 57, 60, 63, 66, 69, 72, 75" + lead="f48_to_f75" elif [ $group = group6 ] ; then fcst_lead="60, 63, 66, 69, 72, 75, 78, 81, 84, 87" + lead="f60_to_f87" fi var=`echo $VAR | tr '[A-Z]' '[a-z]'` @@ -143,6 +139,16 @@ for stat in ets fbias; do verif_type=conus_sfc + save_dir=$DATA/plots/run_${stat}.${score_type}.${valid_time}.${group}.${thresh} + plot_dir=$save_dir/sfc_upper/${valid_beg}-${valid_end} + mkdir -p $plot_dir + mkdir -p $save_dir/data + + echo "#!/bin/ksh" >> run_${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh + echo "export save_dir=$save_dir" >> run_${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh + echo "export log_metplus=$save_dir/log_verif_plotting_job.out" >> run_${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh + echo "export prune_dir=$save_dir/data" >> run_${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh + echo "export PLOT_TYPE=$score_type" >> run_${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh echo "export field=${var}_${level}" >> run_${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh @@ -186,27 +192,17 @@ for stat in ets fbias; do chmod +x run_py.${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh - echo "${DATA}/run_py.${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh" >> run_${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh + echo "${DATA}/scripts/run_py.${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh" >> run_${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh - #Save for restart: + #Tar all files and save for restart: echo "if [ ${score_type} = lead_average ] ; then " >> run_${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh - echo " cp ${plot_dir}/${score_type}_regional_conus_valid_${valid_time}z_tcdc_${stat}_*${thresh}.png $restart/." >> run_${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh + echo " if [ -s ${plot_dir}/${score_type}_regional_conus_valid_${valid_time}z_tcdc_${stat}_*${thresh}.png ] ; then cp ${plot_dir}/${score_type}_regional_conus_valid_${valid_time}z_tcdc_${stat}_*${thresh}.png $restart ; >$restart/run_${stat}.${score_type}.${valid_time}.${group}.${thresh}.completed ; fi " >> run_${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh echo "else" >> run_${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh - echo " cp ${plot_dir}/${score_type}_regional_conus_valid_${valid_time}z_tcdc_${stat}_*.png $restart/." >> run_${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh + echo " if [ -s ${plot_dir}/${score_type}_regional_conus_valid_${valid_time}z_tcdc_${stat}_${lead}.png ] ; then cp ${plot_dir}/${score_type}_regional_conus_valid_${valid_time}z_tcdc_${stat}_${lead}.png $restart ; >$restart/run_${stat}.${score_type}.${valid_time}.${group}.${thresh}.completed ; fi " >> run_${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh echo "fi " >> run_${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh - echo "[[ $? = 0 ]] && >$restart/run_${stat}.${score_type}.${valid_time}.${group}.${thresh}.completed" >> run_${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh chmod +x run_${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh - echo "${DATA}/run_${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh" >> run_all_poe.sh - - else - - #For restart - if [ ${score_type} = lead_average ] ; then - cp $restart/${score_type}_regional_conus_valid_${valid_time}z_tcdc_${stat}_*${thresh}.png ${plot_dir}/. - else - cp $restart/${score_type}_regional_conus_valid_${valid_time}z_tcdc_${stat}_*.png ${plot_dir}/. - fi + echo "${DATA}/scripts/run_${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh" >> run_all_poe.sh fi @@ -227,17 +223,17 @@ chmod +x run_all_poe.sh # Run the POE script in parallel or in sequence order to generate png files # ************************************************************************** if [ $run_mpi = yes ] ; then - mpiexec -np 112 -ppn 112 --cpu-bind verbose,depth cfp ${DATA}/run_all_poe.sh + mpiexec -np 112 -ppn 112 --cpu-bind verbose,depth cfp ${DATA}/scripts/run_all_poe.sh export err=$?; err_chk else - ${DATA}/run_all_poe.sh + ${DATA}/scripts/run_all_poe.sh export err=$?; err_chk fi #************************************************** # Change plot file names to meet the EVS standard #************************************************** -cd $plot_dir +cd $restart var=tcdc levels=L0 @@ -274,11 +270,11 @@ for stat in $stats ; do if [ $score_type = lead_average ] ; then if [ -s ${score_type}_regional_conus_valid_${valid}_${var}_${stat}_${threshold}.png ] ; then - mv ${score_type}_regional_conus_valid_${valid}_${var}_${stat}_${threshold}.png evs.sref.${stat}.${var_level}_${threshold}.last${past_days}days.${scoretype}.valid_${valid}.buk_conus.png + mv ${score_type}_regional_conus_valid_${valid}_${var}_${stat}_${threshold}.png evs.sref.${stat}.${var_level}_${threshold}.last${last_days}days.${scoretype}_valid${valid}.buk_conus.png fi elif [ $score_type = threshold_average ] ; then if [ -s ${score_type}_regional_conus_valid_${valid}_${var}_${stat}_${lead}.png ] ; then - mv ${score_type}_regional_conus_valid_${valid}_${var}_${stat}_${lead}.png evs.sref.${stat}.${var_level}.last${past_days}days.${scoretype}.valid_${valid}.${new_lead}.buk_conus.png + mv ${score_type}_regional_conus_valid_${valid}_${var}_${stat}_${lead}.png evs.sref.${stat}.${var_level}.last${last_days}days.${scoretype}_valid${valid}_${new_lead}.buk_conus.png fi fi @@ -288,16 +284,27 @@ for stat in $stats ; do done done +if [ -s evs*.png ] ; then + tar -cvf evs.plots.sref.cloud.last${last_days}days.v${VDATE}.tar evs*.png +fi -tar -cvf evs.plots.sref.cloud.past${past_days}days.v${VDATE}.tar *.png - +#Cat the plotting log files +log_dir="$DATA/plots" +if [ -s $log_dir/*/log*.out ]; then + log_files=`ls $log_dir/*/log*.out` + for log_file in $log_files ; do + echo "Start: $log_file" + cat "$log_file" + echo "End: $log_file" + done +fi -if [ $SENDCOM = YES ] && [ -s evs.plots.sref.cloud.past${past_days}days.v${VDATE}.tar ] ; then - cp -v evs.plots.sref.cloud.past${past_days}days.v${VDATE}.tar $COMOUTplots/. +if [ $SENDCOM = YES ] && [ -s evs.plots.sref.cloud.last${last_days}days.v${VDATE}.tar ] ; then + cp -v evs.plots.sref.cloud.last${last_days}days.v${VDATE}.tar $COMOUTplots/. fi if [ $SENDDBN = YES ] ; then - $DBNROOT/bin/dbn_alert MODEL EVS_RZDM $job $COMOUTplots/evs.plots.sref.cloud.past${past_days}days.v${VDATE}.tar + $DBNROOT/bin/dbn_alert MODEL EVS_RZDM $job $COMOUTplots/evs.plots.sref.cloud.last${last_days}days.v${VDATE}.tar fi diff --git a/scripts/plots/mesoscale/exevs_mesoscale_sref_cnv_plots.sh b/scripts/plots/mesoscale/exevs_mesoscale_sref_cnv_plots.sh index b71a8ecc8..6080b3dbd 100755 --- a/scripts/plots/mesoscale/exevs_mesoscale_sref_cnv_plots.sh +++ b/scripts/plots/mesoscale/exevs_mesoscale_sref_cnv_plots.sh @@ -10,18 +10,13 @@ set -x export PYTHONPATH=$HOMEevs/ush/$COMPONENT:$PYTHONPATH -cd $DATA +mkdir -p $DATA/scripts +cd $DATA/scripts -export prune_dir=$DATA/data -export save_dir=$DATA/out export output_base_dir=$DATA/stat_archive -export log_metplus=$DATA/logs/GENS_verif_plotting_job.out -mkdir -p $prune_dir -mkdir -p $save_dir mkdir -p $output_base_dir -mkdir -p $DATA/logs -restart=$COMOUTplots/restart/$past_days/sref_cnv_plots +restart=$COMOUTplots/restart/$last_days/sref_cnv_plots if [ ! -d $restart ] ; then mkdir -p $restart fi @@ -39,7 +34,7 @@ model_list='GEFS SREF' models='GEFS, SREF' n=0 -while [ $n -le $past_days ] ; do +while [ $n -le $last_days ] ; do hrs=$((n*24)) first_day=`$NDATE -$hrs ${VDATE}00|cut -c1-8` n=$((n+1)) @@ -49,11 +44,10 @@ export init_beg=$first_day export valid_beg=$first_day #************************************************************************* -# Virtual link the sref's stat data files of past 90 days +# Virtual link the sref's stat data files of last 90 days #************************************************************************** n=0 -while [ $n -le $past_days ] ; do - #hrs=`expr $n \* 24` +while [ $n -le $last_days ] ; do hrs=$((n*24)) day=`$NDATE -$hrs ${VDATE}00|cut -c1-8` echo $day @@ -68,11 +62,6 @@ export fcst_init_hour="0,3,6,9,12,15,18,21" export fcst_valid_hour="0,6,12,18" init_time='init00_to_21z' -export plot_dir=$DATA/out/sfc_upper/${valid_beg}-${valid_end} -if [ ! -d $plot_dir ] ; then - mkdir -p $plot_dir -fi - verif_case=grid2obs line_type='ctc' score_types='lead_average threshold_average' @@ -122,16 +111,22 @@ for VAR in $VARS ; do fcst_lead=" 6, 9, 12, 15, 18, 21, 24, 27, 30, 33, 36, 39, 42, 45, 48, 51, 54, 57, 60, 63, 66, 69, 72, 75, 78, 81, 84, 87" elif [ $group = group1 ] ; then fcst_lead=" 6, 9, 12, 15, 18, 21, 24, 27" + lead="f6-9-12-15-18-21-24-27" elif [ $group = group2 ] ; then fcst_lead="12, 15, 18, 21, 24, 27, 30, 33, 36, 39" + lead="f12_to_f39" elif [ $group = group3 ] ; then fcst_lead="24, 27, 30, 33, 36, 39, 42, 45, 48, 51" + lead="f24_to_f51" elif [ $group = group4 ] ; then fcst_lead="36, 39, 42, 45, 48, 51, 54, 57, 60, 63" + lead="f36_to_f63" elif [ $group = group5 ] ; then fcst_lead="48, 51, 54, 57, 60, 63, 66, 69, 72, 75" + lead="f48_to_f75" elif [ $group = group6 ] ; then fcst_lead="60, 63, 66, 69, 72, 75, 78, 81, 84, 87" + lead="f60_to_f87" fi var=`echo $VAR | tr '[A-Z]' '[a-z]'` @@ -162,6 +157,16 @@ for VAR in $VARS ; do verif_type=conus_sfc + save_dir=$DATA/plots/run_${VAR}.${stat}.${score_type}.${valid_time}.${group}.${thresh} + plot_dir=$save_dir/sfc_upper/${valid_beg}-${valid_end} + mkdir -p $plot_dir + mkdir -p $save_dir/data + + echo "#!/bin/ksh" >> run_${VAR}.${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh + echo "export save_dir=$save_dir" >> run_${VAR}.${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh + echo "export log_metplus=$save_dir/log_verif_plotting_job.out" >> run_${VAR}.${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh + echo "export prune_dir=$save_dir/data" >> run_${VAR}.${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh + echo "export PLOT_TYPE=$score_type" >> run_${VAR}.${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh echo "export field=${var}_${level}" >> run_${VAR}.${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh @@ -181,7 +186,6 @@ for VAR in $VARS ; do echo "export date_type=VALID" >> run_${VAR}.${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh fi - echo "export var_name=$VAR" >> run_${VAR}.${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh echo "export fcts_level=$FCST_LEVEL_value" >> run_${VAR}.${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh echo "export obs_level=$OBS_LEVEL_value" >> run_${VAR}.${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh @@ -209,27 +213,17 @@ for VAR in $VARS ; do chmod +x run_py.${VAR}.${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh - echo "${DATA}/run_py.${VAR}.${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh" >> run_${VAR}.${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh + echo "${DATA}/scripts/run_py.${VAR}.${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh" >> run_${VAR}.${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh - #Save for restart: + #tar all files and save for restart: echo "if [ ${score_type} = lead_average ] ; then " >> run_${VAR}.${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh - echo " cp ${plot_dir}/${score_type}_regional_conus_valid_${valid_time}z_${var_short}_${stat}_*${thresh}.png $restart/." >> run_${VAR}.${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh + echo " if [ -s ${plot_dir}/${score_type}_regional_conus_valid_${valid_time}z_${var_short}_${stat}_*${thresh}.png ] ; then cp ${plot_dir}/${score_type}_regional_conus_valid_${valid_time}z_${var_short}_${stat}_*${thresh}.png $restart ; >$restart/run_${VAR}.${stat}.${score_type}.${valid_time}.${group}.${thresh}.completed ; fi " >> run_${VAR}.${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh echo "else" >> run_${VAR}.${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh - echo " cp ${plot_dir}/${score_type}_regional_conus_valid_${valid_time}z_${var_short}_${stat}_*.png $restart/." >> run_${VAR}.${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh + echo " if [ -s ${plot_dir}/${score_type}_regional_conus_valid_${valid_time}z_${var_short}_${stat}_${lead}.png ] ; then cp ${plot_dir}/${score_type}_regional_conus_valid_${valid_time}z_${var_short}_${stat}_${lead}.png $restart ; >$restart/run_${VAR}.${stat}.${score_type}.${valid_time}.${group}.${thresh}.completed ; fi " >> run_${VAR}.${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh echo "fi " >> run_${VAR}.${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh - echo "[[ $? = 0 ]] && >$restart/run_${VAR}.${stat}.${score_type}.${valid_time}.${group}.${thresh}.completed" >> run_${VAR}.${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh chmod +x run_${VAR}.${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh - echo "${DATA}/run_${VAR}.${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh" >> run_all_poe.sh - - else - - #For restart - if [ ${score_type} = lead_average ] ; then - cp $restart/${score_type}_regional_conus_valid_${valid_time}z_${var_short}_${stat}_*${thresh}.png ${plot_dir}/. - else - cp $restart/${score_type}_regional_conus_valid_${valid_time}z_${var_short}_${stat}_*.png ${plot_dir}/. - fi + echo "${DATA}/scripts/run_${VAR}.${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh" >> run_all_poe.sh fi @@ -251,10 +245,10 @@ chmod +x run_all_poe.sh # Run the POE script in parallel or in sequence order to generate png files # ************************************************************************** if [ $run_mpi = yes ] ; then - mpiexec -np 176 -ppn 88 --cpu-bind verbose,depth cfp ${DATA}/run_all_poe.sh + mpiexec -np 176 -ppn 88 --cpu-bind verbose,depth cfp ${DATA}/scripts/run_all_poe.sh export err=$?; err_chk else - ${DATA}/run_all_poe.sh + ${DATA}/scripts/run_all_poe.sh export err=$?; err_chk fi @@ -263,7 +257,7 @@ echo "run_all_poe done!" #************************************************** # Change plot file names to meet the EVS standard #************************************************** -cd $plot_dir +cd $restart for var in vis hgt ; do levels=L0 @@ -308,11 +302,11 @@ for var in vis hgt ; do if [ $score_type = lead_average ] ; then if [ -s ${score_type}_regional_conus_valid_${valid}_${var}_${stat}_${threshold}.png ] ; then - mv ${score_type}_regional_conus_valid_${valid}_${var}_${stat}_${threshold}.png evs.sref.${stat}.${var_level}_${threshold}.last${past_days}days.${scoretype}.valid_${valid}.buk_conus.png + mv ${score_type}_regional_conus_valid_${valid}_${var}_${stat}_${threshold}.png evs.sref.${stat}.${var_level}_${threshold}.last${last_days}days.${scoretype}_valid${valid}.buk_conus.png fi elif [ $score_type = threshold_average ] ; then if [ -s ${score_type}_regional_conus_valid_${valid}_${var}_${stat}_${lead}.png ] ; then - mv ${score_type}_regional_conus_valid_${valid}_${var}_${stat}_${lead}.png evs.sref.${stat}.${var_level}.last${past_days}days.${scoretype}.valid_${valid}.${new_lead}.buk_conus.png + mv ${score_type}_regional_conus_valid_${valid}_${var}_${stat}_${lead}.png evs.sref.${stat}.${var_level}.last${last_days}days.${scoretype}_valid${valid}.${new_lead}.buk_conus.png fi fi @@ -323,16 +317,28 @@ for var in vis hgt ; do done done -tar -cvf evs.plots.sref.cnv.past${past_days}days.v${VDATE}.tar *.png +if [ -s evs*.png ] ; then + tar -cvf evs.plots.sref.cnv.last${last_days}days.v${VDATE}.tar evs*.png +fi +#Cat the plotting log files +log_dir="$DATA/plots" +if [ -s $log_dir/*/log*.out ]; then + log_files=`ls $log_dir/*/log*.out` + for log_file in $log_files ; do + echo "Start: $log_file" + cat "$log_file" + echo "End: $log_file" + done +fi -if [ $SENDCOM = YES ] && [ -s evs.plots.sref.cnv.past${past_days}days.v${VDATE}.tar ] ; then - cp -v evs.plots.sref.cnv.past${past_days}days.v${VDATE}.tar $COMOUTplots/. +if [ $SENDCOM = YES ] && [ -s evs.plots.sref.cnv.last${last_days}days.v${VDATE}.tar ] ; then + cp -v evs.plots.sref.cnv.last${last_days}days.v${VDATE}.tar $COMOUTplots/. fi if [ $SENDDBN = YES ] ; then - $DBNROOT/bin/dbn_alert MODEL EVS_RZDM $job $COMOUTplots/evs.plots.sref.cnv.past${past_days}days.v${VDATE}.tar + $DBNROOT/bin/dbn_alert MODEL EVS_RZDM $job $COMOUTplots/evs.plots.sref.cnv.last${last_days}days.v${VDATE}.tar fi diff --git a/scripts/plots/mesoscale/exevs_mesoscale_sref_grid2obs_plots.sh b/scripts/plots/mesoscale/exevs_mesoscale_sref_grid2obs_plots.sh index 790bd982b..407497bc2 100755 --- a/scripts/plots/mesoscale/exevs_mesoscale_sref_grid2obs_plots.sh +++ b/scripts/plots/mesoscale/exevs_mesoscale_sref_grid2obs_plots.sh @@ -9,18 +9,13 @@ set -x export PYTHONPATH=$HOMEevs/ush/$COMPONENT:$PYTHONPATH -cd $DATA +mkdir -p $DATA/scripts +cd $DATA/scripts -export prune_dir=$DATA/data -export save_dir=$DATA/out export output_base_dir=$DATA/stat_archive -export log_metplus=$DATA/logs/GENS_verif_plotting_job.out -mkdir -p $prune_dir -mkdir -p $save_dir mkdir -p $output_base_dir -mkdir -p $DATA/logs -restart=$COMOUTplots/restart/$past_days/sref_grid2obs_plots +restart=$COMOUTplots/restart/$last_days/sref_grid2obs_plots if [ ! -d $restart ] ; then mkdir -p $restart fi @@ -37,7 +32,7 @@ model_list='GEFS SREF' models='GEFS, SREF' n=0 -while [ $n -le $past_days ] ; do +while [ $n -le $last_days ] ; do hrs=$((n*24)) first_day=`$NDATE -$hrs ${VDATE}00|cut -c1-8` n=$((n+1)) @@ -48,11 +43,10 @@ export valid_beg=$first_day #************************************************************************* -# Virtual link the sref's stat data files of past 90 days +# Virtual link the sref's stat data files of last 90 days #************************************************************************** n=0 -while [ $n -le $past_days ] ; do - #hrs=`expr $n \* 24` +while [ $n -le $last_days ] ; do hrs=$((n*24)) day=`$NDATE -$hrs ${VDATE}00|cut -c1-8` echo $day @@ -68,11 +62,6 @@ export fcst_valid_hours="0 6 12 18" #valid_time='valid00z_06z_12z_18z' init_time='init00_to_21z' -export plot_dir=$DATA/out/sfc_upper/${valid_beg}-${valid_end} -if [ ! -d $plot_dir ] ; then - mkdir -p $plot_dir -fi - verif_case=$VERIF_CASE #***************************************** # Build a POE file to collect sub-jobs @@ -82,8 +71,6 @@ verif_case=$VERIF_CASE for fcst_valid_hour in $fcst_valid_hours ; do vld=$fcst_valid_hour typeset -Z2 vld - - for stats in crps rmse_spread me ets_fbias; do if [ $stats = crps ] ; then @@ -206,6 +193,16 @@ for fcst_valid_hour in $fcst_valid_hours ; do verif_type=conus_sfc fi + save_dir=$DATA/plots/run_${stats}.${score_type}.${lead}.${VAR}.${FCST_LEVEL_value}.${line_type}.${fcst_valid_hour} + plot_dir=$save_dir/sfc_upper/${valid_beg}-${valid_end} + mkdir -p $plot_dir + mkdir -p $save_dir/data + + echo "#!/bin/ksh" >> run_${stats}.${score_type}.${lead}.${VAR}.${FCST_LEVEL_value}.${line_type}.${fcst_valid_hour}.sh + echo "export save_dir=$save_dir" >> run_${stats}.${score_type}.${lead}.${VAR}.${FCST_LEVEL_value}.${line_type}.${fcst_valid_hour}.sh + echo "export log_metplus=$save_dir/log_verif_plotting_job.out" >> run_${stats}.${score_type}.${lead}.${VAR}.${FCST_LEVEL_value}.${line_type}.${fcst_valid_hour}.sh + echo "export prune_dir=$save_dir/data" >> run_${stats}.${score_type}.${lead}.${VAR}.${FCST_LEVEL_value}.${line_type}.${fcst_valid_hour}.sh + echo "export PLOT_TYPE=$score_type" >> run_${stats}.${score_type}.${lead}.${VAR}.${FCST_LEVEL_value}.${line_type}.${fcst_valid_hour}.sh echo "export field=${var}_${level}" >> run_${stats}.${score_type}.${lead}.${VAR}.${FCST_LEVEL_value}.${line_type}.${fcst_valid_hour}.sh @@ -252,21 +249,24 @@ for fcst_valid_hour in $fcst_valid_hours ; do chmod +x run_py.${stats}.${score_type}.${lead}.${VAR}.${FCST_LEVEL_value}.${line_type}.${fcst_valid_hour}.sh - echo "${DATA}/run_py.${stats}.${score_type}.${lead}.${VAR}.${FCST_LEVEL_value}.${line_type}.${fcst_valid_hour}.sh" >> run_${stats}.${score_type}.${lead}.${VAR}.${FCST_LEVEL_value}.${line_type}.${fcst_valid_hour}.sh - - #Save for restart: - echo "cp ${plot_dir}/${score_type}_regional_conus_valid_${vld}z*.png $restart/." >> run_${stats}.${score_type}.${lead}.${VAR}.${FCST_LEVEL_value}.${line_type}.${fcst_valid_hour}.sh - echo "[[ $? = 0 ]] && >$restart/run_${stats}.${score_type}.${lead}.${VAR}.${FCST_LEVEL_value}.${line_type}.${fcst_valid_hour}.completed" >> run_${stats}.${score_type}.${lead}.${VAR}.${FCST_LEVEL_value}.${line_type}.${fcst_valid_hour}.sh + echo "${DATA}/scripts/run_py.${stats}.${score_type}.${lead}.${VAR}.${FCST_LEVEL_value}.${line_type}.${fcst_valid_hour}.sh" >> run_${stats}.${score_type}.${lead}.${VAR}.${FCST_LEVEL_value}.${line_type}.${fcst_valid_hour}.sh + + #tar all files and save for restart: + echo "if [ ${score_type} = lead_average ] ; then " >> run_${stats}.${score_type}.${lead}.${VAR}.${FCST_LEVEL_value}.${line_type}.${fcst_valid_hour}.sh + echo " if [ -s ${plot_dir}/${score_type}_regional_conus_valid_${vld}z_${level_var}_${stats}*.png ] ; then cp ${plot_dir}/${score_type}_regional_conus_valid_${vld}z_${level_var}_${stats}*.png $restart ; fi " >> run_${stats}.${score_type}.${lead}.${VAR}.${FCST_LEVEL_value}.${line_type}.${fcst_valid_hour}.sh + echo "else" >> run_${stats}.${score_type}.${lead}.${VAR}.${FCST_LEVEL_value}.${line_type}.${fcst_valid_hour}.sh + echo " if [ ${score_type} = threshold_average ] && [ $stats = ets_fbias ] ; then " >> run_${stats}.${score_type}.${lead}.${VAR}.${FCST_LEVEL_value}.${line_type}.${fcst_valid_hour}.sh + echo " if [ -s ${plot_dir}/${score_type}_regional_conus_valid_${vld}z_${level_var}_ets*.png ] ; then cp ${plot_dir}/${score_type}_regional_conus_valid_${vld}z_${level_var}_ets*.png $restart ; fi " >> run_${stats}.${score_type}.${lead}.${VAR}.${FCST_LEVEL_value}.${line_type}.${fcst_valid_hour}.sh + echo " if [ -s ${plot_dir}/${score_type}_regional_conus_valid_${vld}z_${level_var}_fbias*.png ] ; then cp ${plot_dir}/${score_type}_regional_conus_valid_${vld}z_${level_var}_fbias*.png $restart ; fi " >> run_${stats}.${score_type}.${lead}.${VAR}.${FCST_LEVEL_value}.${line_type}.${fcst_valid_hour}.sh + echo " else " >> run_${stats}.${score_type}.${lead}.${VAR}.${FCST_LEVEL_value}.${line_type}.${fcst_valid_hour}.sh + echo " if [ -s ${plot_dir}/${score_type}_regional_conus_valid_${vld}z_${level_var}_${stats}*.png ] ; then cp ${plot_dir}/${score_type}_regional_conus_valid_${vld}z_${level_var}_${stats}*.png $restart ; fi " >> run_${stats}.${score_type}.${lead}.${VAR}.${FCST_LEVEL_value}.${line_type}.${fcst_valid_hour}.sh + echo " fi" >> run_${stats}.${score_type}.${lead}.${VAR}.${FCST_LEVEL_value}.${line_type}.${fcst_valid_hour}.sh + echo "fi" >> run_${stats}.${score_type}.${lead}.${VAR}.${FCST_LEVEL_value}.${line_type}.${fcst_valid_hour}.sh + echo "[[ \$? = 0 ]] && >$restart/run_${stats}.${score_type}.${lead}.${VAR}.${FCST_LEVEL_value}.${line_type}.${fcst_valid_hour}.completed" >> run_${stats}.${score_type}.${lead}.${VAR}.${FCST_LEVEL_value}.${line_type}.${fcst_valid_hour}.sh chmod +x run_${stats}.${score_type}.${lead}.${VAR}.${FCST_LEVEL_value}.${line_type}.${fcst_valid_hour}.sh - echo "${DATA}/run_${stats}.${score_type}.${lead}.${VAR}.${FCST_LEVEL_value}.${line_type}.${fcst_valid_hour}.sh" >> run_all_poe.sh + echo "${DATA}/scripts/run_${stats}.${score_type}.${lead}.${VAR}.${FCST_LEVEL_value}.${line_type}.${fcst_valid_hour}.sh" >> run_all_poe.sh - - else - - #For restart - cp $restart/${score_type}_regional_conus_valid_${vld}z*.png ${plot_dir}/. - fi done #end of line_type @@ -288,10 +288,10 @@ chmod +x run_all_poe.sh # Run the POE script in parallel or in sequence order to generate png files # ************************************************************************** if [ $run_mpi = yes ] ; then - mpiexec -np 216 -ppn 72 --cpu-bind verbose,depth cfp ${DATA}/run_all_poe.sh + mpiexec -np 216 -ppn 72 --cpu-bind verbose,depth cfp ${DATA}/scripts/run_all_poe.sh export err=$?; err_chk else - ${DATA}/run_all_poe.sh + ${DATA}/scripts/run_all_poe.sh export err=$?; err_chk fi @@ -299,7 +299,7 @@ fi #************************************************** # Change plot file names to meet the EVS standard #************************************************** -cd $plot_dir +cd $restart for valid in 00z 06z 12z 18z ; do @@ -365,11 +365,11 @@ for var in hgt tmp ugrd vgrd prmsl rh dpt tcdc cape ; do if [ $unit = mb ] || [ $unit = m ] ; then if [ -s ${score_type}_regional_conus_valid_${valid}_${level}${unit}_${var}_${stat}${end} ] ; then - mv ${score_type}_regional_conus_valid_${valid}_${level}${unit}_${var}_${stat}${end} evs.sref.${stat}.${var_level}.last${past_days}days.${scoretype}_valid_${valid}.buk_conus.png + mv ${score_type}_regional_conus_valid_${valid}_${level}${unit}_${var}_${stat}${end} evs.sref.${stat}.${var_level}.last${last_days}days.${scoretype}_valid${valid}.buk_conus.png fi else if [ -s ${score_type}_regional_conus_valid_${valid}_${var}_${stat}${end} ] ; then - mv ${score_type}_regional_conus_valid_${valid}_${var}_${stat}${end} evs.sref.${stat}.${var_level}.last${past_days}days.${scoretype}_valid_${valid}.buk_conus.png + mv ${score_type}_regional_conus_valid_${valid}_${var}_${stat}${end} evs.sref.${stat}.${var_level}.last${last_days}days.${scoretype}_valid${valid}.buk_conus.png fi fi @@ -381,11 +381,11 @@ for var in hgt tmp ugrd vgrd prmsl rh dpt tcdc cape ; do if [ $unit = m ] ; then if [ -s ${score_type}_regional_conus_valid_${valid}_${level}${unit}_${var}_${stat}_${end} ] ; then - mv ${score_type}_regional_conus_valid_${valid}_${level}${unit}_${var}_${stat}_${end} evs.sref.${stat}.${var_level}.last${past_days}days.${scoretype}_valid_${valid}.buk_conus.png + mv ${score_type}_regional_conus_valid_${valid}_${level}${unit}_${var}_${stat}_${end} evs.sref.${stat}.${var_level}.last${last_days}days.${scoretype}_valid${valid}.buk_conus.png fi else if [ -s ${score_type}_regional_conus_valid_${valid}_${var}_${stat}_${end} ]; then - mv ${score_type}_regional_conus_valid_${valid}_${var}_${stat}_${end} evs.sref.${stat}.${var_level}.last${past_days}days.${scoretype}_valid_${valid}.buk_conus.png + mv ${score_type}_regional_conus_valid_${valid}_${var}_${stat}_${end} evs.sref.${stat}.${var_level}.last${last_days}days.${scoretype}_valid${valid}.buk_conus.png fi fi fi @@ -397,15 +397,27 @@ for var in hgt tmp ugrd vgrd prmsl rh dpt tcdc cape ; do done #vars done #valid -tar -cvf evs.plots.sref.grid2obs.past${past_days}days.v${VDATE}.tar *.png +if [ -s evs*.png ] ; then + tar -cvf evs.plots.sref.grid2obs.last${last_days}days.v${VDATE}.tar evs*.png +fi +#Cat the plotting log files +log_dir="$DATA/plots" +if [ -s $log_dir/*/log*.out ]; then + log_files=`ls $log_dir/*/log*.out` + for log_file in $log_files ; do + echo "Start: $log_file" + cat "$log_file" + echo "End: $log_file" + done +fi -if [ $SENDCOM = YES ] && [ -s evs.plots.sref.grid2obs.past${past_days}days.v${VDATE}.tar ] ; then - cp -v evs.plots.sref.grid2obs.past${past_days}days.v${VDATE}.tar $COMOUTplots/. +if [ $SENDCOM = YES ] && [ -s evs.plots.sref.grid2obs.last${last_days}days.v${VDATE}.tar ] ; then + cp -v evs.plots.sref.grid2obs.last${last_days}days.v${VDATE}.tar $COMOUTplots/. fi if [ $SENDDBN = YES ] ; then - $DBNROOT/bin/dbn_alert MODEL EVS_RZDM $job $COMOUTplots/evs.plots.sref.grid2obs.past${past_days}days.v${VDATE}.tar + $DBNROOT/bin/dbn_alert MODEL EVS_RZDM $job $COMOUTplots/evs.plots.sref.grid2obs.last${last_days}days.v${VDATE}.tar fi diff --git a/scripts/plots/mesoscale/exevs_mesoscale_sref_precip_plots.sh b/scripts/plots/mesoscale/exevs_mesoscale_sref_precip_plots.sh index 6ace9fe30..7d9e27d75 100755 --- a/scripts/plots/mesoscale/exevs_mesoscale_sref_precip_plots.sh +++ b/scripts/plots/mesoscale/exevs_mesoscale_sref_precip_plots.sh @@ -8,18 +8,13 @@ set -x export PYTHONPATH=$HOMEevs/ush/$COMPONENT:$PYTHONPATH -cd $DATA +mkdir $DATA/scripts +cd $DATA/scripts -export prune_dir=$DATA/data -export save_dir=$DATA/out export output_base_dir=$DATA/stat_archive -export log_metplus=$DATA/logs/GENS_verif_plotting_job.out -mkdir -p $prune_dir -mkdir -p $save_dir mkdir -p $output_base_dir -mkdir -p $DATA/logs -restart=$COMOUTplots/restart/$past_days/sref_precip_plots +restart=$COMOUTplots/restart/$last_days/sref_precip_plots if [ ! -d $restart ] ; then mkdir -p $restart fi @@ -37,7 +32,7 @@ model_list='GEFS SREF' models='GEFS, SREF' n=0 -while [ $n -le $past_days ] ; do +while [ $n -le $last_days ] ; do hrs=$((n*24)) first_day=`$NDATE -$hrs ${VDATE}00|cut -c1-8` n=$((n+1)) @@ -47,11 +42,10 @@ export init_beg=$first_day export valid_beg=$first_day #************************************************************************* -# Virtual link the sref's stat data files of past 90 days +# Virtual link the sref's stat data files of last 90 days #************************************************************************** n=0 -while [ $n -le $past_days ] ; do - #hrs=`expr $n \* 24` +while [ $n -le $last_days ] ; do hrs=$((n*24)) day=`$NDATE -$hrs ${VDATE}00|cut -c1-8` echo $day @@ -87,7 +81,15 @@ VARs=APCP_06 FCST_LEVEL_values=A6 for fcst_valid_hour in $fcst_valid_hours ; do - + if [ $fcst_valid_hour = "0,3" ] ; then + valid="00z_03z" + elif [ $fcst_valid_hour = "6,9" ] ; then + valid="06z_09z" + elif [ $fcst_valid_hour = "12,15" ] ; then + valid="12z_15z" + elif [ $fcst_valid_hour = "18,21" ] ; then + valid="18z_21z" + fi for stats in ets fbias fss ; do if [ $stats = ets ] ; then @@ -170,6 +172,16 @@ elif [ $stats = fss ] ; then if [ ! -e $restart/run_${stats}.${score_type}.${lead}.${VAR}.${FCST_LEVEL_value}.${line_type}.${thresh}.${fcst_valid_hour}.completed ] ; then #******************************************************************************************************************* + save_dir=$DATA/plots/run_${stats}.${score_type}.${lead}.${VAR}.${FCST_LEVEL_value}.${line_type}.${thresh}.${fcst_valid_hour} + plot_dir=$save_dir/precip/${valid_beg}-${valid_end} + mkdir -p $plot_dir + mkdir -p $save_dir/data + + echo "#!/bin/ksh" >> run_${stats}.${score_type}.${lead}.${VAR}.${FCST_LEVEL_value}.${line_type}.${thresh}.${fcst_valid_hour}.sh + echo "export save_dir=$save_dir" >> run_${stats}.${score_type}.${lead}.${VAR}.${FCST_LEVEL_value}.${line_type}.${thresh}.${fcst_valid_hour}.sh + echo "export log_metplus=$save_dir/log_verif_plotting_job.out" >> run_${stats}.${score_type}.${lead}.${VAR}.${FCST_LEVEL_value}.${line_type}.${thresh}.${fcst_valid_hour}.sh + echo "export prune_dir=$save_dir/data" >> run_${stats}.${score_type}.${lead}.${VAR}.${FCST_LEVEL_value}.${line_type}.${thresh}.${fcst_valid_hour}.sh + echo "export PLOT_TYPE=$score_type" >> run_${stats}.${score_type}.${lead}.${VAR}.${FCST_LEVEL_value}.${line_type}.${thresh}.${fcst_valid_hour}.sh echo "export field=${var}_${level}" >> run_${stats}.${score_type}.${lead}.${VAR}.${FCST_LEVEL_value}.${line_type}.${thresh}.${fcst_valid_hour}.sh @@ -207,16 +219,19 @@ elif [ $stats = fss ] ; then chmod +x run_py.${stats}.${score_type}.${lead}.${VAR}.${FCST_LEVEL_value}.${line_type}.${thresh}.${fcst_valid_hour}.sh - echo "${DATA}/run_py.${stats}.${score_type}.${lead}.${VAR}.${FCST_LEVEL_value}.${line_type}.${thresh}.${fcst_valid_hour}.sh" >> run_${stats}.${score_type}.${lead}.${VAR}.${FCST_LEVEL_value}.${line_type}.${thresh}.${fcst_valid_hour}.sh + echo "${DATA}/scripts/run_py.${stats}.${score_type}.${lead}.${VAR}.${FCST_LEVEL_value}.${line_type}.${thresh}.${fcst_valid_hour}.sh" >> run_${stats}.${score_type}.${lead}.${VAR}.${FCST_LEVEL_value}.${line_type}.${thresh}.${fcst_valid_hour}.sh #################################################################### - #Save for restart: + #tar all files and save for restart: #################################################################### - echo "cp ${plot_dir}/${score_type}*${stats}*.png $restart/." >> run_${stats}.${score_type}.${lead}.${VAR}.${FCST_LEVEL_value}.${line_type}.${thresh}.${fcst_valid_hour}.sh - echo "[[ $? = 0 ]] && >$restart/run_${stats}.${score_type}.${lead}.${VAR}.${FCST_LEVEL_value}.${line_type}.${thresh}.${fcst_valid_hour}.completed" >> run_${stats}.${score_type}.${lead}.${VAR}.${FCST_LEVEL_value}.${line_type}.${thresh}.${fcst_valid_hour}.sh + echo "if [ ${score_type} = lead_average ] ; then" >> run_${stats}.${score_type}.${lead}.${VAR}.${FCST_LEVEL_value}.${line_type}.${thresh}.${fcst_valid_hour}.sh + echo " if [ -s ${plot_dir}/${score_type}_regional_conus_valid_${valid}_6h_apcp_06_${stats}_*ge${thresh}.png ] ; then cp ${plot_dir}/${score_type}_regional_conus_valid_${valid}_6h_apcp_06_${stats}_*ge${thresh}.png $restart ; >$restart/run_${stats}.${score_type}.${lead}.${VAR}.${FCST_LEVEL_value}.${line_type}.${thresh}.${fcst_valid_hour}.completed ; fi " >> run_${stats}.${score_type}.${lead}.${VAR}.${FCST_LEVEL_value}.${line_type}.${thresh}.${fcst_valid_hour}.sh + echo "else" >> run_${stats}.${score_type}.${lead}.${VAR}.${FCST_LEVEL_value}.${line_type}.${thresh}.${fcst_valid_hour}.sh + echo " if [ -s ${plot_dir}/${score_type}_regional_conus_valid_${valid}_6h_apcp_06_${stats}_f${lead}.png ] ; then cp ${plot_dir}/${score_type}_regional_conus_valid_${valid}_6h_apcp_06_${stats}_f${lead}.png $restart ; >$restart/run_${stats}.${score_type}.${lead}.${VAR}.${FCST_LEVEL_value}.${line_type}.${thresh}.${fcst_valid_hour}.completed ; fi " >> run_${stats}.${score_type}.${lead}.${VAR}.${FCST_LEVEL_value}.${line_type}.${thresh}.${fcst_valid_hour}.sh + echo "fi" >> run_${stats}.${score_type}.${lead}.${VAR}.${FCST_LEVEL_value}.${line_type}.${thresh}.${fcst_valid_hour}.sh chmod +x run_${stats}.${score_type}.${lead}.${VAR}.${FCST_LEVEL_value}.${line_type}.${thresh}.${fcst_valid_hour}.sh - echo "${DATA}/run_${stats}.${score_type}.${lead}.${VAR}.${FCST_LEVEL_value}.${line_type}.${thresh}.${fcst_valid_hour}.sh" >> run_all_poe.sh + echo "${DATA}/scripts/run_${stats}.${score_type}.${lead}.${VAR}.${FCST_LEVEL_value}.${line_type}.${thresh}.${fcst_valid_hour}.sh" >> run_all_poe.sh else @@ -246,17 +261,17 @@ chmod +x run_all_poe.sh # Run the POE script in parallel or in sequence order to generate png files # ************************************************************************** if [ $run_mpi = yes ] ; then - mpiexec -np 100 -ppn 100 --cpu-bind verbose,depth cfp ${DATA}/run_all_poe.sh + mpiexec -np 100 -ppn 100 --cpu-bind verbose,depth cfp ${DATA}/scripts/run_all_poe.sh export err=$?; err_chk else - ${DATA}/run_all_poe.sh + ${DATA}/scripts/run_all_poe.sh export err=$?; err_chk fi #************************************************** # Change plot file names to meet the EVS standard #************************************************** -cd $plot_dir +cd $restart for valid in 00z_03z 06z_09z 12z_15z 18z_21z ; do for stats in ets fbias fss ; do @@ -286,17 +301,17 @@ for stats in ets fbias fss ; do if [ $score_type = lead_average ] ; then if [ $stats = ets ] || [ $stats = fbias ] ; then if [ -s ${score_type}_regional_conus_valid_${valid}_6h_apcp_06_${stats}_${thresh}.png ] ; then - mv ${score_type}_regional_conus_valid_${valid}_6h_apcp_06_${stats}_${thresh}.png evs.sref.${stats}.apcp_6a.${thresh}.last${past_days}days.${scoretype}_valid_${valid}.buk_conus.png + mv ${score_type}_regional_conus_valid_${valid}_6h_apcp_06_${stats}_${thresh}.png evs.sref.${stats}.apcp_6a.${thresh}.last${last_days}days.${scoretype}_valid${valid}.buk_conus.png fi elif [ $stats = fss ] ; then if [ -s ${score_type}_regional_conus_valid_${valid}_6h_apcp_06_${stats}_width1-3-5-7-9-11_${thresh}.png ] ; then - mv ${score_type}_regional_conus_valid_${valid}_6h_apcp_06_${stats}_width1-3-5-7-9-11_${thresh}.png evs.sref.${stats}.apcp_6a.${thresh}.last${past_days}days.${scoretype}_valid_${valid}.buk_conus.png + mv ${score_type}_regional_conus_valid_${valid}_6h_apcp_06_${stats}_width1-3-5-7-9-11_${thresh}.png evs.sref.${stats}.apcp_6a.${thresh}.last${last_days}days.${scoretype}_valid${valid}.buk_conus.png fi fi elif [ $score_type = threshold_average ] ; then for lead in f24 f36 f48 f60 f72 f84 ; do if [ -s ${score_type}_regional_conus_valid_${valid}_6h_apcp_06_${stats}_${lead}.png ] ; then - mv ${score_type}_regional_conus_valid_${valid}_6h_apcp_06_${stats}_${lead}.png evs.sref.${stats}.apcp_6a.last${past_days}days.${scoretype}_valid_${valid}.${lead}.buk_conus.png + mv ${score_type}_regional_conus_valid_${valid}_6h_apcp_06_${stats}_${lead}.png evs.sref.${stats}.apcp_6a.last${last_days}days.${scoretype}_valid${valid}_${lead}.buk_conus.png fi done fi @@ -306,16 +321,29 @@ for stats in ets fbias fss ; do done #stats done #valid -tar -cvf evs.plots.sref.precip.past${past_days}days.v${VDATE}.tar *.png +if [ -s evs*.png ] ; then + tar -cvf evs.plots.sref.precip.last${last_days}days.v${VDATE}.tar evs*.png +fi + +#Cat the plotting log files +log_dir="$DATA/plots" +if [ -s $log_dir/*/log*.out ]; then + log_files=`ls $log_dir/*/log*.out` + for log_file in $log_files ; do + echo "Start: $log_file" + cat "$log_file" + echo "End: $log_file" + done +fi -if [ $SENDCOM = YES ] && [ -s evs.plots.sref.precip.past${past_days}days.v${VDATE}.tar ] ; then - cp -v evs.plots.sref.precip.past${past_days}days.v${VDATE}.tar $COMOUTplots/. +if [ $SENDCOM = YES ] && [ -s evs.plots.sref.precip.last${last_days}days.v${VDATE}.tar ] ; then + cp -v evs.plots.sref.precip.last${last_days}days.v${VDATE}.tar $COMOUTplots/. fi if [ $SENDDBN = YES ] ; then - $DBNROOT/bin/dbn_alert MODEL EVS_RZDM $job $COMOUTplots/evs.plots.sref.precip.past${past_days}days.v${VDATE}.tar + $DBNROOT/bin/dbn_alert MODEL EVS_RZDM $job $COMOUTplots/evs.plots.sref.precip.last${last_days}days.v${VDATE}.tar fi diff --git a/scripts/plots/mesoscale/exevs_mesoscale_sref_precip_spatial_plots.sh b/scripts/plots/mesoscale/exevs_mesoscale_sref_precip_spatial_plots.sh index 6a348659d..f16187ea6 100755 --- a/scripts/plots/mesoscale/exevs_mesoscale_sref_precip_spatial_plots.sh +++ b/scripts/plots/mesoscale/exevs_mesoscale_sref_precip_spatial_plots.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/ksh #******************************************************************************* # Purpose: setup environment, paths, and run the sref precip spatial map # plotting python script @@ -75,7 +75,9 @@ export err=$?; err_chk cd $DATA/grid2grid_plots/plot_output/atmos.${VDATE}/precip/SL1L2_FBAR_24hrAccumMaps_CONUS_precip_spatial_map/images -tar -cvf evs.plots.sref.precip.spatial.map.v${VDATE}.tar *.gif +if [ -s *.gif ] ; then + tar -cvf evs.plots.sref.precip.spatial.map.v${VDATE}.tar *.gif +fi if [ $SENDCOM = YES ] && [ -s evs.plots.sref.precip.spatial.map.v${VDATE}.tar ] ; then cp -v evs.plots.sref.precip.spatial.map.v${VDATE}.tar $COMOUTplots/. diff --git a/scripts/plots/mesoscale/exevs_mesoscale_sref_td2m_plots.sh b/scripts/plots/mesoscale/exevs_mesoscale_sref_td2m_plots.sh index 5674e32ed..c5d93a844 100755 --- a/scripts/plots/mesoscale/exevs_mesoscale_sref_td2m_plots.sh +++ b/scripts/plots/mesoscale/exevs_mesoscale_sref_td2m_plots.sh @@ -8,18 +8,13 @@ set -x export PYTHONPATH=$HOMEevs/ush/$COMPONENT:$PYTHONPATH -cd $DATA +mkdir $DATA/scripts +cd $DATA/scripts -export prune_dir=$DATA/data -export save_dir=$DATA/out export output_base_dir=$DATA/stat_archive -export log_metplus=$DATA/logs/GENS_verif_plotting_job.out -mkdir -p $prune_dir -mkdir -p $save_dir mkdir -p $output_base_dir -mkdir -p $DATA/logs -restart=$COMOUTplots/restart/$past_days/sref_td2m_plots +restart=$COMOUTplots/restart/$last_days/sref_td2m_plots if [ ! -d $restart ] ; then mkdir -p $restart fi @@ -36,7 +31,7 @@ model_list='GEFS SREF' models='GEFS, SREF' n=0 -while [ $n -le $past_days ] ; do +while [ $n -le $last_days ] ; do hrs=$((n*24)) first_day=`$NDATE -$hrs ${VDATE}00|cut -c1-8` n=$((n+1)) @@ -46,11 +41,10 @@ export init_beg=$first_day export valid_beg=$first_day #************************************************************************* -# Virtual link the sref's stat data files of past 90 days +# Virtual link the sref's stat data files of last 90 days #*********************************************************************** n=0 -while [ $n -le $past_days ] ; do - #hrs=`expr $n \* 24` +while [ $n -le $last_days ] ; do hrs=$((n*24)) day=`$NDATE -$hrs ${VDATE}00|cut -c1-8` echo $day @@ -62,15 +56,8 @@ done VX_MASK_LIST="CONUS" export fcst_init_hour="0,3,6,9,12,15,18,21" -#export fcst_valid_hour="0,6,12,18" init_time='init00_to_21z' - -export plot_dir=$DATA/out/sfc_upper/${valid_beg}-${valid_end} -if [ ! -d $plot_dir ] ; then - mkdir -p $plot_dir -fi - verif_case=grid2obs line_type='ctc' score_types='lead_average threshold_average' @@ -90,7 +77,6 @@ for stat in ets fbias; do valid_times="00 06 12 18" thresholds="277.59 283.15 288.71 294.26" export fcst_group=one_group - #export fcst_lead=" 6, 9, 12, 15, 18, 21, 24, 27, 30, 33, 36, 39, 42, 45, 48, 51, 54, 57, 60, 63, 66, 69, 72, 75, 78, 81, 84, 87 elif [ $score_type = threshold_average ] ; then valid_times="00 06 12 18" thresholds=">=277.59,>=283.15,>=288.71,>=294.26" @@ -105,16 +91,22 @@ for stat in ets fbias; do fcst_lead=" 6, 9, 12, 15, 18, 21, 24, 27, 30, 33, 36, 39, 42, 45, 48, 51, 54, 57, 60, 63, 66, 69, 72, 75, 78, 81, 84, 87" elif [ $group = group1 ] ; then fcst_lead=" 6, 9, 12, 15, 18, 21, 24, 27" + lead="f6-9-12-15-18-21-24-27" elif [ $group = group2 ] ; then fcst_lead="12, 15, 18, 21, 24, 27, 30, 33, 36, 39" + lead="f12_to_f39" elif [ $group = group3 ] ; then fcst_lead="24, 27, 30, 33, 36, 39, 42, 45, 48, 51" + lead="f24_to_f51" elif [ $group = group4 ] ; then fcst_lead="36, 39, 42, 45, 48, 51, 54, 57, 60, 63" + lead="f36_to_f63" elif [ $group = group5 ] ; then fcst_lead="48, 51, 54, 57, 60, 63, 66, 69, 72, 75" + lead="f48_to_f75" elif [ $group = group6 ] ; then fcst_lead="60, 63, 66, 69, 72, 75, 78, 81, 84, 87" + lead="f60_to_f87" fi var=`echo $VAR | tr '[A-Z]' '[a-z]'` @@ -145,6 +137,17 @@ for stat in ets fbias; do verif_type=conus_sfc + save_dir=$DATA/plots/run_${stat}.${score_type}.${valid_time}.${group}.${thresh} + plot_dir=$save_dir/sfc_upper/${valid_beg}-${valid_end} + mkdir -p $plot_dir + mkdir -p $save_dir/data + + echo "#!/bin/ksh" >> run_${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh + echo "set -x" >> run_${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh + echo "export save_dir=$save_dir" >> run_${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh + echo "export log_metplus=$save_dir/log_verif_plotting_job.out" >> run_${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh + echo "export prune_dir=$save_dir/data" >> run_${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh + echo "export PLOT_TYPE=$score_type" >> run_${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh echo "export field=${var}_${level}" >> run_${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh @@ -188,19 +191,17 @@ for stat in ets fbias; do chmod +x run_py.${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh - echo "${DATA}/run_py.${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh" >> run_${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh + echo "${DATA}/scripts/run_py.${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh" >> run_${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh #Save for restart: - echo "cp ${plot_dir}/${score_type}*${stat}*.png $restart/." >> run_${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh - echo "[[ $? = 0 ]] && >$restart/run_${stat}.${score_type}.${valid_time}.${group}.${thresh}.completed" >> run_${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh + echo "if [ ${score_type} = lead_average ] ; then " >> run_${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh + echo " if [ -s ${plot_dir}/${score_type}_regional_conus_valid_${valid_time}z_2m_dpt_${stat}_ge${threshold}.png] ; then cp ${plot_dir}/${score_type}_regional_conus_valid_${valid_time}z_2m_dpt_${stat}_ge${threshold}.png $restart ; >$restart/run_${stat}.${score_type}.${valid_time}.${group}.${thresh}.completed ; fi ">>run_${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh + echo "else" >> run_${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh + echo " if [ -s ${plot_dir}/${score_type}_regional_conus_valid_${valid_time}z_2m_dpt_${stat}_${lead}.png ] ; then cp ${plot_dir}/${score_type}_regional_conus_valid_${valid_time}z_2m_dpt_${stat}_${lead}.png $restart ; >$restart/run_${stat}.${score_type}.${valid_time}.${group}.${thresh}.completed ; fi " >> run_${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh + echo "fi" >> run_${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh chmod +x run_${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh - echo "${DATA}/run_${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh" >> run_all_poe.sh - - else - - #For restart - cp $restart/${score_type}*${stat}*.png ${plot_dir}/. + echo "${DATA}/scripts/run_${stat}.${score_type}.${valid_time}.${group}.${thresh}.sh" >> run_all_poe.sh fi @@ -220,17 +221,17 @@ chmod +x run_all_poe.sh # Run the POE script in parallel or in sequence order to generate png files # ************************************************************************** if [ $run_mpi = yes ] ; then - mpiexec -np 80 -ppn 80 --cpu-bind verbose,depth cfp ${DATA}/run_all_poe.sh + mpiexec -np 80 -ppn 80 --cpu-bind verbose,depth cfp ${DATA}/scripts/run_all_poe.sh export err=$?; err_chk else - ${DATA}/run_all_poe.sh + ${DATA}/scripts/run_all_poe.sh export err=$?; err_chk fi #************************************************** # Change plot file names to meet the EVS standard #************************************************** -cd $plot_dir +cd $restart var=dpt levels=z2 @@ -267,11 +268,11 @@ for stat in $stats ; do if [ $score_type = lead_average ] ; then if [ -s ${score_type}_regional_conus_valid_${valid}_2m_dpt_${stat}_${threshold}.png ] ; then - mv ${score_type}_regional_conus_valid_${valid}_2m_dpt_${stat}_${threshold}.png evs.sref.${stat}.${var_level}_${threshold}.last${past_days}days.${scoretype}_valid_${valid}.buk_conus.png + mv ${score_type}_regional_conus_valid_${valid}_2m_dpt_${stat}_${threshold}.png evs.sref.${stat}.${var_level}_${threshold}.last${last_days}days.${scoretype}_valid${valid}.buk_conus.png fi elif [ $score_type = threshold_average ] ; then if [ -s ${score_type}_regional_conus_valid_${valid}_2m_dpt_${stat}_${lead}.png ] ; then - mv ${score_type}_regional_conus_valid_${valid}_2m_dpt_${stat}_${lead}.png evs.sref.${stat}.${var_level}.last${past_days}days.${scoretype}_valid_${valid}.${new_lead}.buk_conus.png + mv ${score_type}_regional_conus_valid_${valid}_2m_dpt_${stat}_${lead}.png evs.sref.${stat}.${var_level}.last${last_days}days.${scoretype}_valid${valid}_${new_lead}.buk_conus.png fi fi @@ -282,15 +283,28 @@ for stat in $stats ; do done -tar -cvf evs.plots.sref.td2m.past${past_days}days.v${VDATE}.tar *.png +if [ -s evs*.png ] ; then + tar -cvf evs.plots.sref.td2m.last${last_days}days.v${VDATE}.tar evs*.png +fi + +#Cat the plotting log files +log_dir="$DATA/plots" +if [ -s $log_dir/*/log*.out ]; then + log_files=`ls $log_dir/*/log*.out` + for log_file in $log_files ; do + echo "Start: $log_file" + cat "$log_file" + echo "End: $log_file" + done +fi -if [ $SENDCOM = YES ] && [ -s evs.plots.sref.td2m.past${past_days}days.v${VDATE}.tar ] ; then - cp -v evs.plots.sref.td2m.past${past_days}days.v${VDATE}.tar $COMOUTplots/. +if [ $SENDCOM = YES ] && [ -s evs.plots.sref.td2m.last${last_days}days.v${VDATE}.tar ] ; then + cp -v evs.plots.sref.td2m.last${last_days}days.v${VDATE}.tar $COMOUTplots/. fi if [ $SENDDBN = YES ] ; then - $DBNROOT/bin/dbn_alert MODEL EVS_RZDM $job $COMOUTplots/evs.plots.sref.td2m.past${past_days}days.v${VDATE}.tar + $DBNROOT/bin/dbn_alert MODEL EVS_RZDM $job $COMOUTplots/evs.plots.sref.td2m.last${last_days}days.v${VDATE}.tar fi diff --git a/scripts/stats/mesoscale/exevs_mesoscale_sref_grid2obs_stats.sh b/scripts/stats/mesoscale/exevs_mesoscale_sref_grid2obs_stats.sh index 6877781c6..6a8c76f38 100755 --- a/scripts/stats/mesoscale/exevs_mesoscale_sref_grid2obs_stats.sh +++ b/scripts/stats/mesoscale/exevs_mesoscale_sref_grid2obs_stats.sh @@ -13,6 +13,7 @@ export machine=${machine:-"WCOSS2"} export PYTHONPATH=$HOMEevs/ush/$COMPONENT:$PYTHONPATH export WORK=$DATA cd $WORK +mkdir -p $WORK/scripts export MET_bin_exec='bin' diff --git a/scripts/stats/mesoscale/exevs_mesoscale_sref_precip_stats.sh b/scripts/stats/mesoscale/exevs_mesoscale_sref_precip_stats.sh index faaa25ee6..e7e1c302f 100755 --- a/scripts/stats/mesoscale/exevs_mesoscale_sref_precip_stats.sh +++ b/scripts/stats/mesoscale/exevs_mesoscale_sref_precip_stats.sh @@ -12,6 +12,7 @@ export machine=${machine:-"WCOSS2"} export PYTHONPATH=$HOMEevs/ush/$COMPONENT:$PYTHONPATH export WORK=$DATA cd $WORK +mkdir -p $WORK/scripts export run_mpi=${run_mpi:-'yes'} export gather=${gather:-'yes'} diff --git a/ush/mesoscale/evs_prepare_sref.sh b/ush/mesoscale/evs_prepare_sref.sh index 629c8fa0a..d3bd0cd53 100755 --- a/ush/mesoscale/evs_prepare_sref.sh +++ b/ush/mesoscale/evs_prepare_sref.sh @@ -3,6 +3,7 @@ # Purpose: Get required input forecast and validation data files # for sref stat jobs # Last update: +# 10/18/2024, resolved the duplicated APCP03 in arw/f03 and 06 members # 06/05/2024, add restart capability, Binbin Zhou Lynker@EMC/NCEP # 05/04/2024, (1) change gfs to gdas for prepbufr files # (2) split the prepbufr files before running METplus PB2NC @@ -36,12 +37,11 @@ if [ $modnam = sref_apcp06 ] && [ ! -e $DATA/sref_mbrs.missing ] ; then #fday -> forecst running day #fvhr -> forecast cycle hour for vvhr in 03 09 15 21 ; do - for fhr in 06 12 18 24 30 36 42 48 54 60 66 72 78 84 ; do + for fhr in 06 12 18 24 30 36 42 48 54 60 66 72 78 84 ; do obsv_vhr=${vday}${vvhr} #validation time: xxxx.tvvhrz.f00 fcst_time=`$ndate -$fhr $obsv_vhr` #fcst running time in yyyyymmddhh export fday=${fcst_time:0:8} export fvhr=${fcst_time:8:2} - export modelpath=${COMINsref}/sref.${fday}/$fvhr/pgrb if [ ! -d $WORK/sref.${fday} ] ; then mkdir $WORK/sref.${fday} fi @@ -51,6 +51,8 @@ if [ $modnam = sref_apcp06 ] && [ ! -e $DATA/sref_mbrs.missing ] ; then mkdir -p $COMOUTrestart/sref.${fday} fi + export modelpath=$WORK/sref.${fday} + for base in arw nmb ; do for mb in ctl n1 n2 n3 n4 n5 n6 p1 p2 p3 p4 p5 p6 ; do @@ -60,16 +62,39 @@ if [ $modnam = sref_apcp06 ] && [ ! -e $DATA/sref_mbrs.missing ] ; then # if yes, copy it to the working directory (restart case) ################################################################################################## if [ ! -s $COMOUTrestart/sref.${fday}/sref_${base}.t${fvhr}z.${mb}.pgrb212.6hr.f${fhr}.nc ] ; then - ${METPLUS_PATH}/ush/run_metplus.py -c ${PARMevs}/metplus_config/machine.conf -c ${PRECIP_CONF}/PcpCombine_fcstSREF_APCP06h.conf - export err=$?; err_chk - if [ -s $output_base/sref_${base}.t${fvhr}z.${mb}.pgrb212.6hr.f${fhr}.nc ] ; then - cp $output_base/sref_${base}.t${fvhr}z.${mb}.pgrb212.6hr.f${fhr}.nc $WORK/sref.${fday}/. - #save for restart: - mv $output_base/sref_${base}.t${fvhr}z.${mb}.pgrb212.6hr.f${fhr}.nc $COMOUTrestart/sref.${fday}/. + + ############################################################################################ + # Note: for arw/f03 and f06 members, the APCP03 is duplicated. So grab the second one (#479:) + # and save the files in the working directory + # ########################################################################################## + if [ $base = arw ] && [ $fhr = 06 ] ; then + sref03=${COMINsref}/sref.${fday}/$fvhr/pgrb/sref_${base}.t${fvhr}z.pgrb212.${mb}.f03.grib2 + if [ -s $sref03 ] ; then + $WGRIB2 $sref03|grep "^479:"|$WGRIB2 -i $sref03 -grib $WORK/sref.${fday}/sref_${base}.t${fvhr}z.pgrb212.${mb}.f03.grib2 + fi + sref06=${COMINsref}/sref.${fday}/$fvhr/pgrb/sref_${base}.t${fvhr}z.pgrb212.${mb}.f06.grib2 + if [ -s $sref06 ] ; then + $WGRIB2 $sref06|grep "^479:"|$WGRIB2 -i $sref06 -grib $WORK/sref.${fday}/sref_${base}.t${fvhr}z.pgrb212.${mb}.f06.grib2 + fi + export modelpath=$WORK/sref.${fday} + else + export modelpath=${COMINsref}/sref.${fday}/$fvhr/pgrb + fi + + if [ -s $modelpath/sref_${base}.t${fvhr}z.pgrb212.${mb}.f${fhr}.grib2 ] ; then + ${METPLUS_PATH}/ush/run_metplus.py -c ${PARMevs}/metplus_config/machine.conf -c ${PRECIP_CONF}/PcpCombine_fcstSREF_APCP06h.conf + export err=$?; err_chk + if [ -s $output_base/sref_${base}.t${fvhr}z.${mb}.pgrb212.6hr.f${fhr}.nc ] ; then + cp $output_base/sref_${base}.t${fvhr}z.${mb}.pgrb212.6hr.f${fhr}.nc $WORK/sref.${fday}/. + #save for restart: + mv $output_base/sref_${base}.t${fvhr}z.${mb}.pgrb212.6hr.f${fhr}.nc $COMOUTrestart/sref.${fday}/. + fi fi else #Restart: - cp $COMOUTrestart/sref.${fday}/sref_${base}.t${fvhr}z.${mb}.pgrb212.6hr.f${fhr}.nc $WORK/sref.${fday} + if [ -s $COMOUTrestart/sref.${fday}/sref_${base}.t${fvhr}z.${mb}.pgrb212.6hr.f${fhr}.nc ] ; then + cp $COMOUTrestart/sref.${fday}/sref_${base}.t${fvhr}z.${mb}.pgrb212.6hr.f${fhr}.nc $WORK/sref.${fday} + fi fi done @@ -104,7 +129,7 @@ if [ $modnam = sref_apcp24_mean ] && [ ! -e $DATA/sref_mbrs.missing ] ; then done done for nfhr in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 ; do - echo $nfhr |$HOMEevs/exec/sref_precip.x + echo $nfhr |$EXECevs/sref_precip.x done export lead='24, 48, 72' @@ -119,7 +144,9 @@ if [ $modnam = sref_apcp24_mean ] && [ ! -e $DATA/sref_mbrs.missing ] ; then if [ ! -d ${COMOUTfinal}/apcp24mean ] ; then mkdir -p ${COMOUTfinal}/apcp24mean fi - cp $output_base/*24mean*.nc ${COMOUTfinal}/apcp24mean + if [ -s $output_base/*24mean*.nc ] ; then + cp $output_base/*24mean*.nc ${COMOUTfinal}/apcp24mean + fi fi fi @@ -251,13 +278,18 @@ export output_base=${WORK}/pb2nc #******************************************************************************* >$WORK/prepbufr.$vday/gdas.t${vhr}z.prepbufr split_by_subset ${COMINobsproc}/gdas.${vday}/$vhr/atmos/gdas.t${vhr}z.prepbufr - cat $WORK/ADPSFC $WORK/SFCSHP $WORK/ADPUPA >> $WORK/prepbufr.$vday/gdas.t${vhr}z.prepbufr - + for subset in ADPSFC SFCSHP ADPUPA ; do + if [ -s ${WORK}/${subset} ]; then + cat ${WORK}/${subset} >> $WORK/prepbufr.$vday/gdas.t${vhr}z.prepbufr + fi + done export bufrpath=$WORK - ${METPLUS_PATH}/ush/run_metplus.py -c ${PARMevs}/metplus_config/machine.conf -c ${GRID2OBS_CONF}/Pb2nc_obsGFS_Prepbufr.conf - export err=$?; err_chk - if [ -s ${WORK}/pb2nc/prepbufr_nc/*.nc ] ; then - cp ${WORK}/pb2nc/prepbufr_nc/*.nc $WORK/prepbufr.${vday} + if [ -s $WORK/prepbufr.$vday/gdas.t${vhr}z.prepbufr ] ; then + ${METPLUS_PATH}/ush/run_metplus.py -c ${PARMevs}/metplus_config/machine.conf -c ${GRID2OBS_CONF}/Pb2nc_obsGFS_Prepbufr.conf + export err=$?; err_chk + if [ -s ${WORK}/pb2nc/prepbufr_nc/*.nc ] ; then + cp ${WORK}/pb2nc/prepbufr_nc/*.nc $WORK/prepbufr.${vday} + fi fi done diff --git a/ush/mesoscale/evs_sref_average_cnv.sh b/ush/mesoscale/evs_sref_average_cnv.sh index 7bc33a36d..d87eb0a30 100755 --- a/ush/mesoscale/evs_sref_average_cnv.sh +++ b/ush/mesoscale/evs_sref_average_cnv.sh @@ -6,14 +6,9 @@ #******************************************************************************************* set -x -#vday=20221018 - typeset -Z2 fhr fhr=$1 - - -for vvfhr in 00 06 12 18 ; do -#for vvfhr in 00 ; do +vvfhr=$2 for n in 2 3 4 5 6 7 8 9 10 11 12 13 ; do ctc25[$n]=0 @@ -24,9 +19,11 @@ for vvfhr in 00 06 12 18 ; do >point_stat_SREF_PREPBUFR_CONUS_FHR${fhr}_CNV_${fhr}0000L_${vday}_${vvfhr}0000V.stat - for base_model in arw nmb ; do +for base_model in arw nmb ; do for mbr in ctl p1 p2 p3 p4 p5 p6 n1 n2 n3 n4 n5 n6 ; do - line=1 + line=1 + + if [ -s point_stat_SREF${base_model}_${mbr}_PREPBUFR_CONUS_FHR${fhr}_${fhr}0000L_${vday}_${vvfhr}0000V.stat ] ; then while read -r LINE; do set -A data $LINE @@ -57,10 +54,8 @@ for vvfhr in 00 06 12 18 ; do fi fi line=$((line+1)) - done < point_stat_SREF${base_model}_${mbr}_PREPBUFR_CONUS_FHR${fhr}_${fhr}0000L_${vday}_${vvfhr}0000V.stat - - done + done < point_stat_SREF${base_model}_${mbr}_PREPBUFR_CONUS_FHR${fhr}_${fhr}0000L_${vday}_${vvfhr}0000V.stat + fi done - done diff --git a/ush/mesoscale/evs_sref_cnv.sh b/ush/mesoscale/evs_sref_cnv.sh index 3c384bd43..b0cf0f314 100755 --- a/ush/mesoscale/evs_sref_cnv.sh +++ b/ush/mesoscale/evs_sref_cnv.sh @@ -10,12 +10,6 @@ set -x export vday=$VDATE export regrid='NONE' -#******************************************** -# Check the input data files availability -# ****************************************** -$USHevs/mesoscale/evs_check_sref_files.sh -export err=$?; err_chk - #******************************************* # Build POE script to collect sub-jobs # ****************************************** @@ -46,67 +40,81 @@ for obsv in prepbufr ; do # Build sub-jobs # First check if the sub-task has been done in the previous run # if yes, skip this sub-task, in this case the sub-task script - # file run_sref_cnv_${fhr}.sh is 0-size in the working directory + # file run_sref_cnv_${fhr}_${vhr}.sh is 0-size in the working directory # otherwise, continue building this sub-task #***************************************************** - for fhr in 3 9 15 21 27 33 39 45 51 57 63 69 75 81 87 ; do - >run_sref_cnv_${fhr}.sh - if [ ! -e $COMOUTrestart/run_sref_cnv_${fhr}.completed ] ; then + cd $WORK/scripts + for vhr in 00 06 12 18 ; do + for fhr in 03 09 15 21 27 33 39 45 51 57 63 69 75 81 87 ; do + + >run_sref_cnv_${fhr}_${vhr}.sh - echo "#!/bin/ksh" >> run_sref_cnv_${fhr}.sh - echo "export output_base=$WORK/grid2obs/run_sref_cnv_${fhr}" >> run_sref_cnv_${fhr}.sh - echo "export domain=CONUS" >> run_sref_cnv_${fhr}.sh + if [ ! -e $COMOUTrestart/run_sref_cnv_${fhr}_${vhr}.completed ] ; then + + ihr=`$NDATE -$fhr $VDATE$vhr|cut -c 9-10` + iday=`$NDATE -$fhr $VDATE$vhr|cut -c 1-8` + input_obsv="$WORK/prepbufr.${VDATE}/prepbufr.t${vhr}z.grid212.nc" + + echo "#!/bin/ksh" >> run_sref_cnv_${fhr}_${vhr}.sh + echo "export output_base=$WORK/grid2obs/run_sref_cnv_${fhr}_${vhr}" >> run_sref_cnv_${fhr}_${vhr}.sh + echo "export domain=CONUS" >> run_sref_cnv_${fhr}_${vhr}.sh - echo "export domain=$domain" >> run_sref_cnv_${fhr}.sh - echo "export obsvhead=$obsv" >> run_sref_cnv_${fhr}.sh - echo "export obsvgrid=grid212" >> run_sref_cnv_${fhr}.sh - echo "export obsvpath=$WORK" >> run_sref_cnv_${fhr}.sh - echo "export vbeg=0" >>run_sref_cnv_${fhr}.sh - echo "export vend=18" >>run_sref_cnv_${fhr}.sh - echo "export valid_increment=21600" >> run_sref_cnv_${fhr}.sh - - echo "export lead=$fhr" >> run_sref_cnv_${fhr}.sh - - echo "export domain=CONUS" >> run_sref_cnv_${fhr}.sh - echo "export model=sref" >> run_sref_cnv_${fhr}.sh - echo "export MODEL=SREF" >> run_sref_cnv_${fhr}.sh - echo "export regrid=NONE " >> run_sref_cnv_${fhr}.sh - echo "export modelhead=sref" >> run_sref_cnv_${fhr}.sh + echo "export domain=$domain" >> run_sref_cnv_${fhr}_${vhr}.sh + echo "export obsvhead=$obsv" >> run_sref_cnv_${fhr}_${vhr}.sh + echo "export obsvgrid=grid212" >> run_sref_cnv_${fhr}_${vhr}.sh + echo "export obsvpath=$WORK" >> run_sref_cnv_${fhr}_${vhr}.sh + echo "export vbeg=$vhr" >>run_sref_cnv_${fhr}_${vhr}.sh + echo "export vend=$vhr" >>run_sref_cnv_${fhr}_${vhr}.sh + echo "export valid_increment=21600" >> run_sref_cnv_${fhr}_${vhr}.sh + + echo "export lead=$fhr" >> run_sref_cnv_${fhr}_${vhr}.sh + + echo "export domain=CONUS" >> run_sref_cnv_${fhr}_${vhr}.sh + echo "export model=sref" >> run_sref_cnv_${fhr}_${vhr}.sh + echo "export MODEL=SREF" >> run_sref_cnv_${fhr}_${vhr}.sh + echo "export regrid=NONE " >> run_sref_cnv_${fhr}_${vhr}.sh + echo "export modelhead=sref" >> run_sref_cnv_${fhr}_${vhr}.sh - echo "export modelpath=$COMINsref" >> run_sref_cnv_${fhr}.sh - echo "export modelgrid=pgrb212" >> run_sref_cnv_${fhr}.sh - echo "export modeltail='.grib2'" >> run_sref_cnv_${fhr}.sh - echo "export extradir=''" >> run_sref_cnv_${fhr}.sh + echo "export modelpath=$COMINsref" >> run_sref_cnv_${fhr}_${vhr}.sh + echo "export modelgrid=pgrb212" >> run_sref_cnv_${fhr}_${vhr}.sh + echo "export modeltail='.grib2'" >> run_sref_cnv_${fhr}_${vhr}.sh + echo "export extradir=''" >> run_sref_cnv_${fhr}_${vhr}.sh export base_model export mbr for base_model in arw nmb ; do for mbr in ctl p1 p2 p3 p4 p5 p6 n1 n2 n3 n4 n5 n6 ; do - echo "export base_model=$base_model" >> run_sref_cnv_${fhr}.sh - echo "export mbr=$mbr" >> run_sref_cnv_${fhr}.sh - echo "${METPLUS_PATH}/ush/run_metplus.py -c ${PARMevs}/metplus_config/machine.conf -c ${GRID2OBS_CONF}/PointStat_fcstSREF_obsPREPBUFR_cnv.conf">> run_sref_cnv_${fhr}.sh + input_fcst=${COMINsref}/sref.${iday}/${ihr}/pgrb/sref_${base_model}.t${ihr}z.pgrb212.${mbr}.f${fhr}.grib2 + if [ -s $input_fcst ] && [ -s $input_obsv ] ; then + echo "export base_model=$base_model" >> run_sref_cnv_${fhr}_${vhr}.sh + echo "export mbr=$mbr" >> run_sref_cnv_${fhr}_${vhr}.sh + echo "${METPLUS_PATH}/ush/run_metplus.py -c ${PARMevs}/metplus_config/machine.conf -c ${GRID2OBS_CONF}/PointStat_fcstSREF_obsPREPBUFR_cnv.conf">> run_sref_cnv_${fhr}_${vhr}.sh + export err=$?; err_chk + fi done done - echo "cd \$output_base/stat" >> run_sref_cnv_${fhr}.sh - echo "$USHevs/mesoscale/evs_sref_average_cnv.sh $fhr" >> run_sref_cnv_${fhr}.sh + echo "cd \$output_base/stat" >> run_sref_cnv_${fhr}_${vhr}.sh + echo "$USHevs/mesoscale/evs_sref_average_cnv.sh $fhr $vhr" >> run_sref_cnv_${fhr}_${vhr}.sh + echo "export err=\$?; err_chk" >> run_sref_cnv_${fhr}_${vhr}.sh - #echo "rm \$output_base/stat/*SREFarw*.stat ">> run_sref_cnv_${fhr}.sh - #echo "rm \$output_base/stat/*SREFnmb*.stat ">> run_sref_cnv_${fhr}.sh + #echo "rm \$output_base/stat/*SREFarw*.stat ">> run_sref_cnv_${fhr}_${vhr}.sh + #echo "rm \$output_base/stat/*SREFnmb*.stat ">> run_sref_cnv_${fhr}_${vhr}.sh - echo "if [ -s \$output_base/stat/*CNV*.stat ] ; then" >> run_sref_cnv_${fhr}.sh - echo " cp \$output_base/stat/*CNV*.stat $COMOUTsmall" >> run_sref_cnv_${fhr}.sh - echo "fi" >> run_sref_cnv_${fhr}.sh + echo "if [ -s \$output_base/stat/*CNV*.stat ] ; then" >> run_sref_cnv_${fhr}_${vhr}.sh + echo " cp \$output_base/stat/*CNV*.stat $COMOUTsmall" >> run_sref_cnv_${fhr}_${vhr}.sh + echo "fi" >> run_sref_cnv_${fhr}_${vhr}.sh #For restart: - echo "[[ \$? = 0 ]] && >$COMOUTrestart/run_sref_cnv_${fhr}.completed" >> run_sref_cnv_${fhr}.sh + echo "[[ \$? = 0 ]] && >$COMOUTrestart/run_sref_cnv_${fhr}_${vhr}.completed" >> run_sref_cnv_${fhr}_${vhr}.sh - chmod +x run_sref_cnv_${fhr}.sh - echo "${DATA}/run_sref_cnv_${fhr}.sh" >> run_all_sref_cnv_poe.sh + chmod +x run_sref_cnv_${fhr}_${vhr}.sh + echo "${DATA}/scripts/run_sref_cnv_${fhr}_${vhr}.sh" >> run_all_sref_cnv_poe.sh fi # check restart for the sub-job + done done done @@ -116,9 +124,9 @@ done #************************************************* chmod 775 run_all_sref_cnv_poe.sh if [ $run_mpi = yes ] ; then - mpiexec -n 15 -ppn 15 --cpu-bind core --depth=2 cfp ${DATA}/run_all_sref_cnv_poe.sh + mpiexec -n 15 -ppn 15 --cpu-bind verbose,core cfp ${DATA}/scripts/run_all_sref_cnv_poe.sh else - ${DATA}/run_all_sref_cnv_poe.sh + ${DATA}/scripts/run_all_sref_cnv_poe.sh fi export err=$?; err_chk @@ -126,22 +134,4 @@ if [ $? = 0 ] ; then >$COMOUTrestart/evs_sref_cnv.completed fi -echo "Print stat generation metplus log files begin:" -log_dirs="$DATA/grid2obs/*/logs" -for log_dir in $log_dirs; do - if [ -d $log_dir ]; then - log_file_count=$(find $log_dir -type f | wc -l) - if [[ $log_file_count -ne 0 ]]; then - log_files=("$log_dir"/*) - for log_file in "${log_files[@]}"; do - if [ -f "$log_file" ]; then - echo "Start: $log_file" - cat "$log_file" - echo "End: $log_file" - fi - done - fi - fi -done -echo "Print stat generation metplus log files end" diff --git a/ush/mesoscale/evs_sref_gather.sh b/ush/mesoscale/evs_sref_gather.sh index 4a30ab52b..6c6387516 100755 --- a/ush/mesoscale/evs_sref_gather.sh +++ b/ush/mesoscale/evs_sref_gather.sh @@ -24,6 +24,7 @@ MODEL=`echo $modnam | tr '[a-z]' '[A-Z]'` #************************************************ # Build sub-jobs # *********************************************** +cd $WORK/scripts >run_gather_${verify}.sh echo "export output_base=${WORK}/gather" >> run_gather_${verify}.sh @@ -41,38 +42,24 @@ MODEL=`echo $modnam | tr '[a-z]' '[A-Z]'` if [ $verify = grid2obs ] ; then echo "${METPLUS_PATH}/ush/run_metplus.py -c ${PARMevs}/metplus_config/machine.conf -c ${GRID2OBS_CONF}/StatAnlysis_fcstSREF_obsPREPBUFR_GatherByDay.conf " >> run_gather_${verify}.sh + echo "export err=$?; err_chk" >> run_gather_${verify}.sh elif [ $verify = precip ] ; then echo "${METPLUS_PATH}/ush/run_metplus.py -c ${PARMevs}/metplus_config/machine.conf -c ${PRECIP_CONF}/StatAnlysis_fcstSREF_obsCCPA_GatherByDay.conf " >> run_gather_${verify}.sh + echo "export err=$?; err_chk" >> run_gather_${verify}.sh fi echo "[[ $SENDCOM = YES && -s ${WORK}/gather/${vday}/${modnam}_${verify}_${vday}.stat ]] && cp -v ${WORK}/gather/${vday}/${modnam}_${verify}_${vday}.stat $COMOUTfinal/evs.stats.${model}.${verify}.v${vday}.stat">>run_gather_${verify}.sh chmod +x run_gather_${verify}.sh - echo "${DATA}/run_gather_${verify}.sh" >> run_gather_all_poe.sh + echo "${DATA}/scripts/run_gather_${verify}.sh" >> run_gather_all_poe.sh chmod +x run_gather_all_poe.sh #******************************************** # Run the POE script #******************************************* -${DATA}/run_gather_all_poe.sh +${DATA}/scripts/run_gather_all_poe.sh export err=$?; err_chk -echo "Print stat gather metplus log files begin:" -log_dir="$DATA/gather/logs" -if [ -d $log_dir ]; then - log_file_count=$(find $log_dir -type f | wc -l) - if [[ $log_file_count -ne 0 ]]; then - log_files=("$log_dir"/*) - for log_file in "${log_files[@]}"; do - if [ -f "$log_file" ]; then - echo "Start: $log_file" - cat "$log_file" - echo "End: $log_file" - fi - done - fi -fi -echo "Print stat gather metplus log files end" diff --git a/ush/mesoscale/evs_sref_grid2obs.sh b/ush/mesoscale/evs_sref_grid2obs.sh index a819be45b..7e20dd111 100755 --- a/ush/mesoscale/evs_sref_grid2obs.sh +++ b/ush/mesoscale/evs_sref_grid2obs.sh @@ -10,12 +10,6 @@ set -x export vday=$VDATE export regrid='NONE' -#******************************************** -# Check the input data files availability -# ****************************************** -$USHevs/mesoscale/evs_check_sref_files.sh -export err=$?; err_chk - #******************************************* # Build POE script to collect sub-jobs # ****************************************** @@ -40,45 +34,51 @@ for obsv in prepbufr ; do export err=$?; err_chk else #Restart: copy saved stat files from previous runs - cp -r $COMOUTrestart/prepbufr.${VDATE} $WORK/. + if [ -d $COMOUTrestart/prepbufr.${VDATE} ] ; then + cp -r $COMOUTrestart/prepbufr.${VDATE} $WORK/. + fi fi #***************************************** # Build sub-jobs #***************************************** - for fhr in fhr1 fhr2 ; do - >run_sref_g2o_${domain}.${obsv}.${fhr}.sh + cd $WORK/scripts + for vhr in 00 06 12 18 ; do + for fhr in 03 09 15 21 27 33 39 45 51 57 63 69 75 81 87 ; do + >run_sref_g2o_${domain}.${obsv}.${fhr}.${vhr}.sh + + if [ ! -e $COMOUTrestart/run_sref_g2o_${domain}.${obsv}.${fhr}.${vhr}.completed ] ; then - if [ ! -e $COMOUTrestart/run_sref_g2o_${domain}.${obsv}.${fhr}.completed ] ; then + ihr=`$NDATE -$fhr $VDATE$vhr|cut -c 9-10` + iday=`$NDATE -$fhr $VDATE$vhr|cut -c 1-8` + input_fcst=${COMINsref}/sref.${iday}/${ihr}/pgrb/sref_???.t${ihr}z.pgrb212.*.f${fhr}.grib2 + input_obsv="$WORK/prepbufr.${VDATE}/prepbufr.t${vhr}z.grid212.nc" - echo "#!/bin/ksh" >> run_sref_g2o_${domain}.${obsv}.${fhr}.sh - echo "export output_base=$WORK/grid2obs/${domain}.${obsv}.${fhr}" >> run_sref_g2o_${domain}.${obsv}.${fhr}.sh - echo "export domain=CONUS" >> run_sref_g2o_${domain}.${obsv}.${fhr}.sh + if [ -s $input_fcst ] && [ -s $input_obsv ] ; then + + echo "#!/bin/ksh" >> run_sref_g2o_${domain}.${obsv}.${fhr}.${vhr}.sh + echo "export output_base=$WORK/grid2obs/${domain}.${obsv}.${fhr}.${vhr}" >> run_sref_g2o_${domain}.${obsv}.${fhr}.${vhr}.sh + echo "export domain=CONUS" >> run_sref_g2o_${domain}.${obsv}.${fhr}.${vhr}.sh - echo "export domain=$domain" >> run_sref_g2o_${domain}.${obsv}.${fhr}.sh - echo "export obsvhead=$obsv" >> run_sref_g2o_${domain}.${obsv}.${fhr}.sh - echo "export obsvgrid=grid212" >> run_sref_g2o_${domain}.${obsv}.${fhr}.sh - echo "export obsvpath=$WORK" >> run_sref_g2o_${domain}.${obsv}.${fhr}.sh - echo "export vbeg=0" >>run_sref_g2o_${domain}.${obsv}.${fhr}.sh - echo "export vend=18" >>run_sref_g2o_${domain}.${obsv}.${fhr}.sh - echo "export valid_increment=21600" >> run_sref_g2o_${domain}.${obsv}.${fhr}.sh - if [ $fhr = fhr1 ] ; then - echo "export lead='3, 9, 15, 21, 27, 33, 39'" >> run_sref_g2o_${domain}.${obsv}.${fhr}.sh - elif [ $fhr = fhr2 ] ; then - echo "export lead='45, 51, 57, 63, 69, 75, 81, 87'" >> run_sref_g2o_${domain}.${obsv}.${fhr}.sh - fi - - echo "export domain=CONUS" >> run_sref_g2o_${domain}.${obsv}.${fhr}.sh - echo "export model=sref" >> run_sref_g2o_${domain}.${obsv}.${fhr}.sh - echo "export MODEL=SREF" >> run_sref_g2o_${domain}.${obsv}.${fhr}.sh - echo "export regrid=NONE " >> run_sref_g2o_${domain}.${obsv}.${fhr}.sh - echo "export modelhead=sref" >> run_sref_g2o_${domain}.${obsv}.${fhr}.sh + echo "export domain=$domain" >> run_sref_g2o_${domain}.${obsv}.${fhr}.${vhr}.sh + echo "export obsvhead=$obsv" >> run_sref_g2o_${domain}.${obsv}.${fhr}.${vhr}.sh + echo "export obsvgrid=grid212" >> run_sref_g2o_${domain}.${obsv}.${fhr}.${vhr}.sh + echo "export obsvpath=$WORK" >> run_sref_g2o_${domain}.${obsv}.${fhr}.${vhr}.sh + echo "export vbeg=$vhr" >>run_sref_g2o_${domain}.${obsv}.${fhr}.${vhr}.sh + echo "export vend=$vhr" >>run_sref_g2o_${domain}.${obsv}.${fhr}.${vhr}.sh + echo "export valid_increment=21600" >> run_sref_g2o_${domain}.${obsv}.${fhr}.${vhr}.sh + echo "export lead=$fhr" >> run_sref_g2o_${domain}.${obsv}.${fhr}.${vhr}.sh + echo "export domain=CONUS" >> run_sref_g2o_${domain}.${obsv}.${fhr}.${vhr}.sh + echo "export model=sref" >> run_sref_g2o_${domain}.${obsv}.${fhr}.${vhr}.sh + echo "export MODEL=SREF" >> run_sref_g2o_${domain}.${obsv}.${fhr}.${vhr}.sh + echo "export regrid=NONE " >> run_sref_g2o_${domain}.${obsv}.${fhr}.${vhr}.sh + echo "export modelhead=sref" >> run_sref_g2o_${domain}.${obsv}.${fhr}.${vhr}.sh - echo "export modelpath=$COMINsref" >> run_sref_g2o_${domain}.${obsv}.${fhr}.sh - echo "export modelmean=$EVSINsrefmean" >> run_sref_g2o_${domain}.${obsv}.${fhr}.sh - echo "export modelgrid=pgrb212" >> run_sref_g2o_${domain}.${obsv}.${fhr}.sh - echo "export modeltail='.grib2'" >> run_sref_g2o_${domain}.${obsv}.${fhr}.sh - echo "export extradir=''" >> run_sref_g2o_${domain}.${obsv}.${fhr}.sh + echo "export modelpath=$COMINsref" >> run_sref_g2o_${domain}.${obsv}.${fhr}.${vhr}.sh + echo "export modelmean=$EVSINsrefmean" >> run_sref_g2o_${domain}.${obsv}.${fhr}.${vhr}.sh + echo "export modelgrid=pgrb212" >> run_sref_g2o_${domain}.${obsv}.${fhr}.${vhr}.sh + echo "export modeltail='.grib2'" >> run_sref_g2o_${domain}.${obsv}.${fhr}.${vhr}.sh + echo "export extradir=''" >> run_sref_g2o_${domain}.${obsv}.${fhr}.${vhr}.sh ########################################################################################################### # Adding following "if blocks" for restart capability: @@ -87,46 +87,53 @@ for obsv in prepbufr ; do # 3. if any one of the 4 exits, skip it. But for gneensprod, all of the nc files generated from previous run # are copied back to the output_base/stat directory # ########################################################################################################### - echo "if [ ! -e $COMOUTrestart/run_sref_g2o_genensprod_${domain}.${obsv}.${fhr}.completed ] ; then " >> run_sref_g2o_${domain}.${obsv}.${fhr}.sh - echo " ${METPLUS_PATH}/ush/run_metplus.py -c ${PARMevs}/metplus_config/machine.conf -c ${GRID2OBS_CONF}/GenEnsProd_fcstSREF_obsPREPBUFR.conf " >> run_sref_g2o_${domain}.${obsv}.${fhr}.sh - echo " if [ -s \$output_base/stat/GenEnsProd_SREF_PREPBUFR*.nc ] ; then" >> run_sref_g2o_${domain}.${obsv}.${fhr}.sh - echo " cp \$output_base/stat/GenEnsProd_SREF_PREPBUFR*.nc $COMOUTrestart" >> run_sref_g2o_${domain}.${obsv}.${fhr}.sh - echo " fi " >> run_sref_g2o_${domain}.${obsv}.${fhr}.sh - echo " [[ \$? = 0 ]] && >$COMOUTrestart/run_sref_g2o_genensprod_${domain}.${obsv}.${fhr}.completed" >> run_sref_g2o_${domain}.${obsv}.${fhr}.sh - echo "else " >> run_sref_g2o_${domain}.${obsv}.${fhr}.sh - echo " mkdir -p \$output_base/stat" >> run_sref_g2o_${domain}.${obsv}.${fhr}.sh - echo " cp $COMOUTrestart/GenEnsProd_SREF_PREPBUFR*.nc \$output_base/stat" >> run_sref_g2o_${domain}.${obsv}.${fhr}.sh - echo "fi" >> run_sref_g2o_${domain}.${obsv}.${fhr}.sh - - echo "if [ ! -e $COMOUTrestart/run_sref_g2o_ens_${domain}.${obsv}.${fhr}.completed ] ; then " >> run_sref_g2o_${domain}.${obsv}.${fhr}.sh - echo " ${METPLUS_PATH}/ush/run_metplus.py -c ${PARMevs}/metplus_config/machine.conf -c ${GRID2OBS_CONF}/EnsembleStat_fcstSREF_obsPREPBUFR.conf " >> run_sref_g2o_${domain}.${obsv}.${fhr}.sh - echo " [[ \$? = 0 ]] && >$COMOUTrestart/run_sref_g2o_ens_${domain}.${obsv}.${fhr}.completed" >> run_sref_g2o_${domain}.${obsv}.${fhr}.sh - echo "fi " >> run_sref_g2o_${domain}.${obsv}.${fhr}.sh - - echo "if [ ! -e $COMOUTrestart/run_sref_g2o_mean_${domain}.${obsv}.${fhr}.completed ] ; then " >> run_sref_g2o_${domain}.${obsv}.${fhr}.sh - echo " ${METPLUS_PATH}/ush/run_metplus.py -c ${PARMevs}/metplus_config/machine.conf -c ${GRID2OBS_CONF}/PointStat_fcstSREF_obsPREPBUFR_mean.conf">> run_sref_g2o_${domain}.${obsv}.${fhr}.sh - echo " [[ \$? = 0 ]] && >$COMOUTrestart/run_sref_g2o_mean_${domain}.${obsv}.${fhr}.completed" >> run_sref_g2o_${domain}.${obsv}.${fhr}.sh - echo "fi " >> run_sref_g2o_${domain}.${obsv}.${fhr}.sh - - echo "if [ ! -e $COMOUTrestart/run_sref_g2o_prob_${domain}.${obsv}.${fhr}.completed ] ; then " >> run_sref_g2o_${domain}.${obsv}.${fhr}.sh - echo " ${METPLUS_PATH}/ush/run_metplus.py -c ${PARMevs}/metplus_config/machine.conf -c ${GRID2OBS_CONF}/PointStat_fcstSREF_obsPREPBUFR_prob.conf">> run_sref_g2o_${domain}.${obsv}.${fhr}.sh - echo " [[ \$? = 0 ]] && >$COMOUTrestart/run_sref_g2o_prob_${domain}.${obsv}.${fhr}.completed" >> run_sref_g2o_${domain}.${obsv}.${fhr}.sh - echo "fi " >> run_sref_g2o_${domain}.${obsv}.${fhr}.sh - - echo "if [ -s \$output_base/stat/*.stat ] ; then" >> run_sref_g2o_${domain}.${obsv}.${fhr}.sh - echo " cp \$output_base/stat/*.stat $COMOUTsmall" >> run_sref_g2o_${domain}.${obsv}.${fhr}.sh - echo "fi" >> run_sref_g2o_${domain}.${obsv}.${fhr}.sh + echo "if [ ! -e $COMOUTrestart/run_sref_g2o_genensprod_${domain}.${obsv}.${fhr}.${vhr}.completed ] ; then " >> run_sref_g2o_${domain}.${obsv}.${fhr}.${vhr}.sh + echo " ${METPLUS_PATH}/ush/run_metplus.py -c ${PARMevs}/metplus_config/machine.conf -c ${GRID2OBS_CONF}/GenEnsProd_fcstSREF_obsPREPBUFR.conf " >> run_sref_g2o_${domain}.${obsv}.${fhr}.${vhr}.sh + echo " export err=\$?; err_chk" >> run_sref_g2o_${domain}.${obsv}.${fhr}.${vhr}.sh + echo " if [ -s \$output_base/stat/GenEnsProd_SREF_PREPBUFR*.nc ] ; then" >> run_sref_g2o_${domain}.${obsv}.${fhr}.${vhr}.sh + echo " cp \$output_base/stat/GenEnsProd_SREF_PREPBUFR*.nc $COMOUTrestart" >> run_sref_g2o_${domain}.${obsv}.${fhr}.${vhr}.sh + echo " fi " >> run_sref_g2o_${domain}.${obsv}.${fhr}.${vhr}.sh + echo " [[ \$? = 0 ]] && >$COMOUTrestart/run_sref_g2o_genensprod_${domain}.${obsv}.${fhr}.${vhr}.completed" >> run_sref_g2o_${domain}.${obsv}.${fhr}.${vhr}.sh + echo "else " >> run_sref_g2o_${domain}.${obsv}.${fhr}.${vhr}.sh + echo " mkdir -p \$output_base/stat" >> run_sref_g2o_${domain}.${obsv}.${fhr}.${vhr}.sh + echo " if [ -s $COMOUTrestart/GenEnsProd_SREF_PREPBUFR*.nc ] ; then" >> run_sref_g2o_${domain}.${obsv}.${fhr}.${vhr}.sh + echo " cp $COMOUTrestart/GenEnsProd_SREF_PREPBUFR*.nc \$output_base/stat" >> run_sref_g2o_${domain}.${obsv}.${fhr}.${vhr}.sh + echo " fi" >> run_sref_g2o_${domain}.${obsv}.${fhr}.${vhr}.sh + echo "fi" >> run_sref_g2o_${domain}.${obsv}.${fhr}.${vhr}.sh + + echo "if [ ! -e $COMOUTrestart/run_sref_g2o_ens_${domain}.${obsv}.${fhr}.${vhr}.completed ] ; then " >> run_sref_g2o_${domain}.${obsv}.${fhr}.${vhr}.sh + echo " ${METPLUS_PATH}/ush/run_metplus.py -c ${PARMevs}/metplus_config/machine.conf -c ${GRID2OBS_CONF}/EnsembleStat_fcstSREF_obsPREPBUFR.conf " >> run_sref_g2o_${domain}.${obsv}.${fhr}.${vhr}.sh + echo " export err=\$?; err_chk" >> run_sref_g2o_${domain}.${obsv}.${fhr}.${vhr}.sh + echo " [[ \$? = 0 ]] && >$COMOUTrestart/run_sref_g2o_ens_${domain}.${obsv}.${fhr}.${vhr}.completed" >> run_sref_g2o_${domain}.${obsv}.${fhr}.${vhr}.sh + echo "fi " >> run_sref_g2o_${domain}.${obsv}.${fhr}.${vhr}.sh + + echo "if [ ! -e $COMOUTrestart/run_sref_g2o_mean_${domain}.${obsv}.${fhr}.${vhr}.completed ] ; then " >> run_sref_g2o_${domain}.${obsv}.${fhr}.${vhr}.sh + echo " [[ -s \$output_base/stat/GenEnsProd_SREF_PREPBUFR_${domain}_FHR${fhr}_${VDATE}_${vhr}0000V_ens.nc ]] && ${METPLUS_PATH}/ush/run_metplus.py -c ${PARMevs}/metplus_config/machine.conf -c ${GRID2OBS_CONF}/PointStat_fcstSREF_obsPREPBUFR_mean.conf">> run_sref_g2o_${domain}.${obsv}.${fhr}.${vhr}.sh + echo " export err=\$?; err_chk" >> run_sref_g2o_${domain}.${obsv}.${fhr}.${vhr}.sh + echo " [[ \$? = 0 ]] && >$COMOUTrestart/run_sref_g2o_mean_${domain}.${obsv}.${fhr}.${vhr}.completed" >> run_sref_g2o_${domain}.${obsv}.${fhr}.${vhr}.sh + echo "fi " >> run_sref_g2o_${domain}.${obsv}.${fhr}.${vhr}.sh + + echo "if [ ! -e $COMOUTrestart/run_sref_g2o_prob_${domain}.${obsv}.${fhr}.${vhr}.completed ] ; then " >> run_sref_g2o_${domain}.${obsv}.${fhr}.${vhr}.sh + echo " [[ -s \$output_base/stat/GenEnsProd_SREF_PREPBUFR_${domain}_FHR${fhr}_${VDATE}_${vhr}0000V_ens.nc ]] && ${METPLUS_PATH}/ush/run_metplus.py -c ${PARMevs}/metplus_config/machine.conf -c ${GRID2OBS_CONF}/PointStat_fcstSREF_obsPREPBUFR_prob.conf">> run_sref_g2o_${domain}.${obsv}.${fhr}.${vhr}.sh + echo " export err=\$?; err_chk" >> run_sref_g2o_${domain}.${obsv}.${fhr}.${vhr}.sh + echo " [[ \$? = 0 ]] && >$COMOUTrestart/run_sref_g2o_prob_${domain}.${obsv}.${fhr}.${vhr}.completed" >> run_sref_g2o_${domain}.${obsv}.${fhr}.${vhr}.sh + echo "fi " >> run_sref_g2o_${domain}.${obsv}.${fhr}.${vhr}.sh + + echo "if [ -s \$output_base/stat/*.stat ] ; then" >> run_sref_g2o_${domain}.${obsv}.${fhr}.${vhr}.sh + echo " cp \$output_base/stat/*.stat $COMOUTsmall" >> run_sref_g2o_${domain}.${obsv}.${fhr}.${vhr}.sh + echo "fi" >> run_sref_g2o_${domain}.${obsv}.${fhr}.${vhr}.sh #Mark that all of the 4 METplus processes are completed for next restart run: - echo "[[ \$? = 0 ]] && >$COMOUTrestart/run_sref_g2o_${domain}.${obsv}.${fhr}.completed" >>run_sref_g2o_${domain}.${obsv}.${fhr}.sh + echo "[[ \$? = 0 ]] && >$COMOUTrestart/run_sref_g2o_${domain}.${obsv}.${fhr}.${vhr}.completed" >>run_sref_g2o_${domain}.${obsv}.${fhr}.${vhr}.sh - chmod +x run_sref_g2o_${domain}.${obsv}.${fhr}.sh - echo "${DATA}/run_sref_g2o_${domain}.${obsv}.${fhr}.sh" >> run_all_sref_g2o_poe.sh + chmod +x run_sref_g2o_${domain}.${obsv}.${fhr}.${vhr}.sh + echo "${DATA}/scripts/run_sref_g2o_${domain}.${obsv}.${fhr}.${vhr}.sh" >> run_all_sref_g2o_poe.sh - fi # check restart for the sub-job + fi + fi # check restart for the sub-job done - + done done #*************************************************** @@ -134,31 +141,12 @@ done #************************************************* chmod 775 run_all_sref_g2o_poe.sh if [ $run_mpi = yes ] ; then - mpiexec -n 2 -ppn 2 --cpu-bind core --depth=2 cfp ${DATA}/run_all_sref_g2o_poe.sh + mpiexec -n 15 -ppn 15 --cpu-bind verbose,core cfp ${DATA}/scripts/run_all_sref_g2o_poe.sh else - ${DATA}/run_all_sref_g2o_poe.sh + ${DATA}/scripts/run_all_sref_g2o_poe.sh fi export err=$?; err_chk -echo "Print stat generation metplus log files begin:" -log_dirs="$DATA/grid2obs/*/logs" -for log_dir in $log_dirs; do - if [ -d $log_dir ]; then - log_file_count=$(find $log_dir -type f | wc -l) - if [[ $log_file_count -ne 0 ]]; then - log_files=("$log_dir"/*) - for log_file in "${log_files[@]}"; do - if [ -f "$log_file" ]; then - echo "Start: $log_file" - cat "$log_file" - echo "End: $log_file" - fi - done - fi - fi -done -echo "Print stat generation metplus log files end" - #*********************************************** # Gather small stat files to forma big stat file # ********************************************** diff --git a/ush/mesoscale/evs_sref_precip.sh b/ush/mesoscale/evs_sref_precip.sh index 88766f983..05c76d988 100755 --- a/ush/mesoscale/evs_sref_precip.sh +++ b/ush/mesoscale/evs_sref_precip.sh @@ -9,25 +9,17 @@ set -x export vday=$VDATE export regrid='NONE' -############################################################ - -#******************************************** -# Check the input data files availability -# ****************************************** -$USHevs/mesoscale/evs_check_sref_files.sh -export err=$?; err_chk #******************************************* # Build POE script to collect sub-jobs # ****************************************** +cd $WORK/scripts >run_all_sref_precip_poe export model=sref for obsv in ccpa ; do -#####for obsv in ndas ccpa ; do - export domain=CONUS #*********************************************** @@ -56,9 +48,10 @@ for obsv in ccpa ; do #******************************************************* # Build sub-jobs #***************************************************** - for fhr in fhr1 fhr2 ; do + for vhr in 03 09 15 21 ; do + for fhr in 06 12 18 24 30 36 42 48 54 60 66 72 78 84 ; do - >run_sref_mpi_${domain}.${obsv}.${fhr}.sh + >run_sref_mpi_${domain}.${obsv}.${fhr}.${vhr}.sh ############################################################################################################# # Adding following "if blocks" for restart capability: @@ -67,96 +60,95 @@ for obsv in ccpa ; do # 3. if any one of the 5 exits, skip it. But for genensprod, all of the nc files generated from previous run # are copied back to the output_base/stat directory ############################################################################################################### - if [ ! -e $COMOUTrestart/run_sref_mpi_${domain}.${obsv}.${fhr}.completed ] ; then + if [ ! -e $COMOUTrestart/run_sref_mpi_${domain}.${obsv}.${fhr}.${vhr}.completed ] ; then - echo "#!/bin/ksh" >> run_sref_mpi_${domain}.${obsv}.${fhr}.sh - echo "export output_base=$WORK/precip/${domain}.${obsv}.${fhr}" >> run_sref_mpi_${domain}.${obsv}.${fhr}.sh + ihr=`$NDATE -$fhr $VDATE$vhr|cut -c 9-10` + iday=`$NDATE -$fhr $VDATE$vhr|cut -c 1-8` + input_fcst="${WORK}/sref.${iday}/sref_???.t${ihr}z.*.pgrb212.6hr.f${fhr}.nc" + input_obsv212="$WORK/ccpa.${VDATE}/ccpa.t${vhr}z.grid212.06h.f00.nc" + input_obsv240="$WORK/ccpa.${VDATE}/ccpa.t${vhr}z.grid240.06h.f00.nc" + if [ -s $input_fcst ] && [ -s $input_obsv212 ] && [ -s $input_obsv240 ] ; then + echo "#!/bin/ksh" >> run_sref_mpi_${domain}.${obsv}.${fhr}.${vhr}.sh + echo "export output_base=$WORK/precip/${domain}.${obsv}.${fhr}.${vhr}" >> run_sref_mpi_${domain}.${obsv}.${fhr}.${vhr}.sh - echo "export obsvhead=$obsv" >> run_sref_mpi_${domain}.${obsv}.${fhr}.sh - echo "export obsvpath=$WORK" >> run_sref_mpi_${domain}.${obsv}.${fhr}.sh - echo "export vbeg=03" >>run_sref_mpi_${domain}.${obsv}.${fhr}.sh - echo "export vend=21" >>run_sref_mpi_${domain}.${obsv}.${fhr}.sh - echo "export valid_increment=21600" >> run_sref_mpi_${domain}.${obsv}.${fhr}.sh - - if [ $fhr = fhr1 ] ; then - echo "export lead='6, 12, 18, 24, 30, 36, 42'" >> run_sref_mpi_${domain}.${obsv}.${fhr}.sh - elif [ $fhr = fhr2 ] ; then - echo "export lead='48, 54, 60, 66, 72, 78, 84'" >> run_sref_mpi_${domain}.${obsv}.${fhr}.sh - fi - - echo "export domain=CONUS" >> run_sref_mpi_${domain}.${obsv}.${fhr}.sh - echo "export model=sref" >> run_sref_mpi_${domain}.${obsv}.${fhr}.sh - echo "export MODEL=SREF" >> run_sref_mpi_${domain}.${obsv}.${fhr}.sh - echo "export regrid=NONE " >> run_sref_mpi_${domain}.${obsv}.${fhr}.sh - echo "export modelhead=sref" >> run_sref_mpi_${domain}.${obsv}.${fhr}.sh - - if [ $obsv = ccpa ] ; then - echo "export modelpath=$WORK" >> run_sref_mpi_${domain}.${obsv}.${fhr}.sh - echo "export modelgrid=pgrb212.6hr" >> run_sref_mpi_${domain}.${obsv}.${fhr}.sh - echo "export modeltail='.nc'" >> run_sref_mpi_${domain}.${obsv}.${fhr}.sh - echo "export extradir=''" >> run_sref_mpi_${domain}.${obsv}.${fhr}.sh - - else - echo "export modelpath=$COMINsref" >> run_sref_mpi_${domain}.${obsv}.${fhr}.sh - echo "export modelgrid=pgrb212" >> run_sref_mpi_${domain}.${obsv}.${fhr}.sh - echo "export modeltail='.grib2'" >> run_sref_mpi_${domain}.${obsv}.${fhr}.sh - echo "export extradir=''" >> run_sref_mpi_${domain}.${obsv}.${fhr}.sh - - fi - - echo "export grid=G212" >> run_sref_mpi_${domain}.${obsv}.${fhr}.sh - echo "export obsvgrid=grid212" >> run_sref_mpi_${domain}.${obsv}.${fhr}.sh + echo "export obsvhead=$obsv" >> run_sref_mpi_${domain}.${obsv}.${fhr}.${vhr}.sh + echo "export obsvpath=$WORK" >> run_sref_mpi_${domain}.${obsv}.${fhr}.${vhr}.sh + echo "export vbeg=$vhr" >>run_sref_mpi_${domain}.${obsv}.${fhr}.${vhr}.sh + echo "export vend=$vhr" >>run_sref_mpi_${domain}.${obsv}.${fhr}.${vhr}.sh + echo "export valid_increment=21600" >> run_sref_mpi_${domain}.${obsv}.${fhr}.${vhr}.sh + echo "export lead=$fhr" >> run_sref_mpi_${domain}.${obsv}.${fhr}.${vhr}.sh + echo "export domain=CONUS" >> run_sref_mpi_${domain}.${obsv}.${fhr}.${vhr}.sh + echo "export model=sref" >> run_sref_mpi_${domain}.${obsv}.${fhr}.${vhr}.sh + echo "export MODEL=SREF" >> run_sref_mpi_${domain}.${obsv}.${fhr}.${vhr}.sh + echo "export regrid=NONE " >> run_sref_mpi_${domain}.${obsv}.${fhr}.${vhr}.sh + echo "export modelhead=sref" >> run_sref_mpi_${domain}.${obsv}.${fhr}.${vhr}.sh + + echo "export modelpath=$WORK" >> run_sref_mpi_${domain}.${obsv}.${fhr}.${vhr}.sh + echo "export modelgrid=pgrb212.6hr" >> run_sref_mpi_${domain}.${obsv}.${fhr}.${vhr}.sh + echo "export modeltail='.nc'" >> run_sref_mpi_${domain}.${obsv}.${fhr}.${vhr}.sh + echo "export extradir=''" >> run_sref_mpi_${domain}.${obsv}.${fhr}.${vhr}.sh + + echo "export grid=G212" >> run_sref_mpi_${domain}.${obsv}.${fhr}.${vhr}.sh + echo "export obsvgrid=grid212" >> run_sref_mpi_${domain}.${obsv}.${fhr}.${vhr}.sh #Adding following 5 "if-blocks" for restart capability: - echo "if [ ! -e $COMOUTrestart/${domain}.${obsv}.${fhr}.GenEnsProd.completed ] ; then " >> run_sref_mpi_${domain}.${obsv}.${fhr}.sh - echo " ${METPLUS_PATH}/ush/run_metplus.py -c ${PARMevs}/metplus_config/machine.conf -c ${PRECIP_CONF}/GenEnsProd_fcstSREF_obsCCPA_G212.conf " >> run_sref_mpi_${domain}.${obsv}.${fhr}.sh - echo " if [ -s \$output_base/stat/GenEnsProd_SREF_CCPA*.nc ] ; then" >> run_sref_mpi_${domain}.${obsv}.${fhr}.sh - echo " cp \$output_base/stat/GenEnsProd_SREF_CCPA*.nc $COMOUTrestart" >> run_sref_mpi_${domain}.${obsv}.${fhr}.sh - echo " [[ \$? = 0 ]] && >$COMOUTrestart/${domain}.${obsv}.${fhr}.GenEnsProd.completed" >> run_sref_mpi_${domain}.${obsv}.${fhr}.sh - echo " fi " >> run_sref_mpi_${domain}.${obsv}.${fhr}.sh - echo "else" >> run_sref_mpi_${domain}.${obsv}.${fhr}.sh - echo " mkdir -p \$output_base/stat" >> run_sref_mpi_${domain}.${obsv}.${fhr}.sh - echo " cp $COMOUTrestart/GenEnsProd_SREF_CCPA*.nc \$output_base/stat" >> run_sref_mpi_${domain}.${obsv}.${fhr}.sh - echo "fi" >> run_sref_mpi_${domain}.${obsv}.${fhr}.sh - - echo "if [ ! -e $COMOUTrestart/${domain}.${obsv}.${fhr}.EnsembleStat.completed ] ; then" >> run_sref_mpi_${domain}.${obsv}.${fhr}.sh - echo " ${METPLUS_PATH}/ush/run_metplus.py -c ${PARMevs}/metplus_config/machine.conf -c ${PRECIP_CONF}/EnsembleStat_fcstSREF_obsCCPA_G212.conf " >> run_sref_mpi_${domain}.${obsv}.${fhr}.sh - echo " [[ \$? = 0 ]] && >$COMOUTrestart/${domain}.${obsv}.${fhr}.EnsembleStat.completed" >> run_sref_mpi_${domain}.${obsv}.${fhr}.sh - echo "fi " >> run_sref_mpi_${domain}.${obsv}.${fhr}.sh - - echo "if [ ! -e $COMOUTrestart/${domain}.${obsv}.${fhr}.GridStat_mean.completed ] ; then" >> run_sref_mpi_${domain}.${obsv}.${fhr}.sh - echo " ${METPLUS_PATH}/ush/run_metplus.py -c ${PARMevs}/metplus_config/machine.conf -c ${PRECIP_CONF}/GridStat_fcstSREF_obsCCPA_mean_G212.conf " >> run_sref_mpi_${domain}.${obsv}.${fhr}.sh - echo " [[ \$? = 0 ]] && >$COMOUTrestart/${domain}.${obsv}.${fhr}.GridStat_mean.completed " >> run_sref_mpi_${domain}.${obsv}.${fhr}.sh - echo "fi " >> run_sref_mpi_${domain}.${obsv}.${fhr}.sh - - echo "if [ ! -e $COMOUTrestart/${domain}.${obsv}.${fhr}.GridStat_prob.completed ] ; then" >> run_sref_mpi_${domain}.${obsv}.${fhr}.sh - echo "${METPLUS_PATH}/ush/run_metplus.py -c ${PARMevs}/metplus_config/machine.conf -c ${PRECIP_CONF}/GridStat_fcstSREF_obsCCPA_prob_G212.conf " >> run_sref_mpi_${domain}.${obsv}.${fhr}.sh - echo " [[ \$? = 0 ]] && >$COMOUTrestart/${domain}.${obsv}.${fhr}.GridStat_prob.completed " >> run_sref_mpi_${domain}.${obsv}.${fhr}.sh - echo "fi " >> run_sref_mpi_${domain}.${obsv}.${fhr}.sh - - echo "if [ ! -e $COMOUTrestart/${domain}.${obsv}.${fhr}.GridStat_mean_G240.completed ] ; then" >> run_sref_mpi_${domain}.${obsv}.${fhr}.sh - echo " export obsvgrid=grid240" >> run_sref_mpi_${domain}.${obsv}.${fhr}.sh - echo " export grid=G240" >> run_sref_mpi_${domain}.${obsv}.${fhr}.sh - echo " export regrid=OBS" >> run_sref_mpi_${domain}.${obsv}.${fhr}.sh - echo " ${METPLUS_PATH}/ush/run_metplus.py -c ${PARMevs}/metplus_config/machine.conf -c ${PRECIP_CONF}/GridStat_fcstSREF_obsCCPA_mean_G240.conf " >> run_sref_mpi_${domain}.${obsv}.${fhr}.sh - echo " [[ \$? = 0 ]] && >$COMOUTrestart/${domain}.${obsv}.${fhr}.GridStat_mean_G240.completed" >> run_sref_mpi_${domain}.${obsv}.${fhr}.sh - echo "fi " >> run_sref_mpi_${domain}.${obsv}.${fhr}.sh - - echo "if [ -s \$output_base/stat/*.stat ] ; then" >> run_sref_mpi_${domain}.${obsv}.${fhr}.sh - echo " cp \$output_base/stat/*.stat $COMOUTsmall" >> run_sref_mpi_${domain}.${obsv}.${fhr}.sh - echo "fi" >> run_sref_mpi_${domain}.${obsv}.${fhr}.sh + echo "if [ ! -e $COMOUTrestart/${domain}.${obsv}.${fhr}.${vhr}.GenEnsProd.completed ] ; then " >> run_sref_mpi_${domain}.${obsv}.${fhr}.${vhr}.sh + echo " ${METPLUS_PATH}/ush/run_metplus.py -c ${PARMevs}/metplus_config/machine.conf -c ${PRECIP_CONF}/GenEnsProd_fcstSREF_obsCCPA_G212.conf " >> run_sref_mpi_${domain}.${obsv}.${fhr}.${vhr}.sh + echo " export err=\$?; err_chk" >> run_sref_mpi_${domain}.${obsv}.${fhr}.${vhr}.sh + echo " if [ -s \$output_base/stat/GenEnsProd_SREF_CCPA*.nc ] ; then" >> run_sref_mpi_${domain}.${obsv}.${fhr}.${vhr}.sh + echo " cp \$output_base/stat/GenEnsProd_SREF_CCPA*.nc $COMOUTrestart" >> run_sref_mpi_${domain}.${obsv}.${fhr}.${vhr}.sh + echo " [[ \$? = 0 ]] && >$COMOUTrestart/${domain}.${obsv}.${fhr}.${vhr}.GenEnsProd.completed" >> run_sref_mpi_${domain}.${obsv}.${fhr}.${vhr}.sh + echo " fi " >> run_sref_mpi_${domain}.${obsv}.${fhr}.${vhr}.sh + echo "else" >> run_sref_mpi_${domain}.${obsv}.${fhr}.${vhr}.sh + echo " mkdir -p \$output_base/stat" >> run_sref_mpi_${domain}.${obsv}.${fhr}.${vhr}.sh + echo " if [ -s $COMOUTrestart/GenEnsProd_SREF_CCPA*.nc ] ; then " >> run_sref_mpi_${domain}.${obsv}.${fhr}.${vhr}.sh + echo " cp $COMOUTrestart/GenEnsProd_SREF_CCPA*.nc \$output_base/stat" >> run_sref_mpi_${domain}.${obsv}.${fhr}.${vhr}.sh + echo " fi" >> run_sref_mpi_${domain}.${obsv}.${fhr}.${vhr}.sh + echo "fi" >> run_sref_mpi_${domain}.${obsv}.${fhr}.${vhr}.sh + + echo "if [ ! -e $COMOUTrestart/${domain}.${obsv}.${fhr}.${vhr}.EnsembleStat.completed ] ; then" >> run_sref_mpi_${domain}.${obsv}.${fhr}.${vhr}.sh + echo " ${METPLUS_PATH}/ush/run_metplus.py -c ${PARMevs}/metplus_config/machine.conf -c ${PRECIP_CONF}/EnsembleStat_fcstSREF_obsCCPA_G212.conf " >> run_sref_mpi_${domain}.${obsv}.${fhr}.${vhr}.sh + echo " export err=\$?; err_chk" >> run_sref_mpi_${domain}.${obsv}.${fhr}.${vhr}.sh + echo " [[ \$? = 0 ]] && >$COMOUTrestart/${domain}.${obsv}.${fhr}.${vhr}.EnsembleStat.completed" >> run_sref_mpi_${domain}.${obsv}.${fhr}.${vhr}.sh + echo "fi " >> run_sref_mpi_${domain}.${obsv}.${fhr}.${vhr}.sh + + echo "if [ ! -e $COMOUTrestart/${domain}.${obsv}.${fhr}.${vhr}.GridStat_mean.completed ] ; then" >> run_sref_mpi_${domain}.${obsv}.${fhr}.${vhr}.sh + echo " [[ -s \$output_base/stat/GenEnsProd_SREF_CCPA_G212_FHR${fhr}_${VDATE}_${vhr}0000V_ens.nc ]] && ${METPLUS_PATH}/ush/run_metplus.py -c ${PARMevs}/metplus_config/machine.conf -c ${PRECIP_CONF}/GridStat_fcstSREF_obsCCPA_mean_G212.conf " >> run_sref_mpi_${domain}.${obsv}.${fhr}.${vhr}.sh + echo " [[ \$? = 0 ]] && >$COMOUTrestart/${domain}.${obsv}.${fhr}.${vhr}.GridStat_mean.completed " >> run_sref_mpi_${domain}.${obsv}.${fhr}.${vhr}.sh + echo " export err=\$?; err_chk" >> run_sref_mpi_${domain}.${obsv}.${fhr}.${vhr}.sh + echo "fi " >> run_sref_mpi_${domain}.${obsv}.${fhr}.${vhr}.sh + + echo "if [ ! -e $COMOUTrestart/${domain}.${obsv}.${fhr}.${vhr}.GridStat_prob.completed ] ; then" >> run_sref_mpi_${domain}.${obsv}.${fhr}.${vhr}.sh + echo " [[ -s \$output_base/stat/GenEnsProd_SREF_CCPA_G212_FHR${fhr}_${VDATE}_${vhr}0000V_ens.nc ]] && ${METPLUS_PATH}/ush/run_metplus.py -c ${PARMevs}/metplus_config/machine.conf -c ${PRECIP_CONF}/GridStat_fcstSREF_obsCCPA_prob_G212.conf " >> run_sref_mpi_${domain}.${obsv}.${fhr}.${vhr}.sh + echo " export err=\$?; err_chk" >> run_sref_mpi_${domain}.${obsv}.${fhr}.${vhr}.sh + echo " [[ \$? = 0 ]] && >$COMOUTrestart/${domain}.${obsv}.${fhr}.${vhr}.GridStat_prob.completed " >> run_sref_mpi_${domain}.${obsv}.${fhr}.${vhr}.sh + echo "fi " >> run_sref_mpi_${domain}.${obsv}.${fhr}.${vhr}.sh + + echo "if [ ! -e $COMOUTrestart/${domain}.${obsv}.${fhr}.${vhr}.GridStat_mean_G240.completed ] ; then" >> run_sref_mpi_${domain}.${obsv}.${fhr}.${vhr}.sh + echo " export obsvgrid=grid240" >> run_sref_mpi_${domain}.${obsv}.${fhr}.${vhr}.sh + echo " export grid=G240" >> run_sref_mpi_${domain}.${obsv}.${fhr}.${vhr}.sh + echo " export regrid=OBS" >> run_sref_mpi_${domain}.${obsv}.${fhr}.${vhr}.sh + echo " [[ -s \$output_base/stat/GenEnsProd_SREF_CCPA_G212_FHR${fhr}_${VDATE}_${vhr}0000V_ens.nc ]] && ${METPLUS_PATH}/ush/run_metplus.py -c ${PARMevs}/metplus_config/machine.conf -c ${PRECIP_CONF}/GridStat_fcstSREF_obsCCPA_mean_G240.conf " >> run_sref_mpi_${domain}.${obsv}.${fhr}.${vhr}.sh + echo " export err=\$?; err_chk" >> run_sref_mpi_${domain}.${obsv}.${fhr}.${vhr}.sh + echo " [[ \$? = 0 ]] && >$COMOUTrestart/${domain}.${obsv}.${fhr}.${vhr}.GridStat_mean_G240.completed" >> run_sref_mpi_${domain}.${obsv}.${fhr}.${vhr}.sh + echo "fi " >> run_sref_mpi_${domain}.${obsv}.${fhr}.${vhr}.sh + + echo "if [ -s \$output_base/stat/*.stat ] ; then" >> run_sref_mpi_${domain}.${obsv}.${fhr}.${vhr}.sh + echo " cp \$output_base/stat/*.stat $COMOUTsmall" >> run_sref_mpi_${domain}.${obsv}.${fhr}.${vhr}.sh + echo "fi" >> run_sref_mpi_${domain}.${obsv}.${fhr}.${vhr}.sh #Mark that all of the 5 METplus processes are completed for next restart run: - echo "[[ \$? = 0 ]] && >$COMOUTrestart/run_sref_mpi_${domain}.${obsv}.${fhr}.completed" >> run_sref_mpi_${domain}.${obsv}.${fhr}.sh + echo "[[ \$? = 0 ]] && >$COMOUTrestart/run_sref_mpi_${domain}.${obsv}.${fhr}.${vhr}.completed" >> run_sref_mpi_${domain}.${obsv}.${fhr}.${vhr}.sh - chmod +x run_sref_mpi_${domain}.${obsv}.${fhr}.sh - echo "${DATA}/run_sref_mpi_${domain}.${obsv}.${fhr}.sh" >> run_all_sref_precip_poe + chmod +x run_sref_mpi_${domain}.${obsv}.${fhr}.${vhr}.sh + echo "${DATA}/scripts/run_sref_mpi_${domain}.${obsv}.${fhr}.${vhr}.sh" >> run_all_sref_precip_poe + fi fi # check restart for the sub-job done - + done done chmod +x run_all_sref_precip_poe @@ -165,31 +157,12 @@ chmod +x run_all_sref_precip_poe # Run POE script to get small stat files #************************************************* if [ $run_mpi = yes ] ; then - mpiexec -n 4 -ppn 4 --cpu-bind core --depth=2 cfp ${DATA}/run_all_sref_precip_poe + mpiexec -n 28 -ppn 28 --cpu-bind verbose,core cfp ${DATA}/scripts/run_all_sref_precip_poe else - ${DATA}/run_all_sref_precip_poe + ${DATA}/scripts/run_all_sref_precip_poe fi export err=$?; err_chk -echo "Print stat generation metplus log files begin:" -log_dirs="$DATA/precip/*/logs" -for log_dir in $log_dirs; do - if [ -d $log_dir ]; then - log_file_count=$(find $log_dir -type f | wc -l) - if [[ $log_file_count -ne 0 ]]; then - log_files=("$log_dir"/*) - for log_file in "${log_files[@]}"; do - if [ -f "$log_file" ]; then - echo "Start: $log_file" - cat "$log_file" - echo "End: $log_file" - fi - done - fi - fi -done -echo "Print stat generation metplus log files end" - #*********************************************** # Gather small stat files to forma big stat file # ********************************************** diff --git a/ush/mesoscale/ush_sref_plot_cnv_py/prune_stat_files.py b/ush/mesoscale/ush_sref_plot_cnv_py/prune_stat_files.py index 882ba852a..643ee1389 100755 --- a/ush/mesoscale/ush_sref_plot_cnv_py/prune_stat_files.py +++ b/ush/mesoscale/ush_sref_plot_cnv_py/prune_stat_files.py @@ -100,10 +100,10 @@ def prune_data(data_dir, prune_dir, tmp_dir, output_base_template, valid_range, ) # Prune the MET .stat files and write to new file for met_stat_file in met_stat_files: - ps = subprocess.Popen('grep -R "'+model+'" '+met_stat_file+filter_cmd, - shell=True, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, encoding='UTF-8') - grep_output = ps.communicate()[0] + grep = subprocess.run('grep -R "'+model+'" '+met_stat_file+filter_cmd, + shell=True, capture_output=True, encoding="utf8") + grep_output = grep.stdout + all_grep_output = all_grep_output+grep_output pruned_met_stat_file = os.path.join(pruned_data_dir, model+'.stat') diff --git a/ush/mesoscale/ush_sref_plot_precip_py/prune_stat_files.py b/ush/mesoscale/ush_sref_plot_precip_py/prune_stat_files.py index 882ba852a..7133cb47a 100755 --- a/ush/mesoscale/ush_sref_plot_precip_py/prune_stat_files.py +++ b/ush/mesoscale/ush_sref_plot_precip_py/prune_stat_files.py @@ -100,10 +100,10 @@ def prune_data(data_dir, prune_dir, tmp_dir, output_base_template, valid_range, ) # Prune the MET .stat files and write to new file for met_stat_file in met_stat_files: - ps = subprocess.Popen('grep -R "'+model+'" '+met_stat_file+filter_cmd, - shell=True, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, encoding='UTF-8') - grep_output = ps.communicate()[0] + grep = subprocess.run('grep -R "'+model+'" '+met_stat_file+filter_cmd, + shell=True, capture_output=True, encoding="utf8") + grep_output = grep.stdout + all_grep_output = all_grep_output+grep_output pruned_met_stat_file = os.path.join(pruned_data_dir, model+'.stat') diff --git a/ush/mesoscale/ush_sref_plot_precip_py/sref_atmos_util.py b/ush/mesoscale/ush_sref_plot_precip_py/sref_atmos_util.py index c817e0b3c..751db90dd 100644 --- a/ush/mesoscale/ush_sref_plot_precip_py/sref_atmos_util.py +++ b/ush/mesoscale/ush_sref_plot_precip_py/sref_atmos_util.py @@ -1889,13 +1889,12 @@ def condense_model_stat_files(logger, input_dir, output_file, model, obs, ) for model_stat_file in model_stat_files: logger.debug(f"Getting data from {model_stat_file}") - ps = subprocess.Popen( - 'grep -R "'+model+' " '+model_stat_file+grep_opts, - shell=True, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, encoding='UTF-8' - ) + grep = subprocess.run('grep -R "'+model+'" '+model_stat_file+grep_opts, + shell=True, capture_output=True, encoding="utf8") logger.debug(f"Ran {ps.args}") - all_grep_output = all_grep_output+ps.communicate()[0] + + all_grep_output = all_grep_output+grep.stdout + logger.debug(f"Condensed {model} .stat file at " +f"{output_file}") with open(output_file, 'w') as f: diff --git a/ush/mesoscale/ush_sref_plot_py/prune_stat_files.py b/ush/mesoscale/ush_sref_plot_py/prune_stat_files.py index 882ba852a..7133cb47a 100755 --- a/ush/mesoscale/ush_sref_plot_py/prune_stat_files.py +++ b/ush/mesoscale/ush_sref_plot_py/prune_stat_files.py @@ -100,10 +100,10 @@ def prune_data(data_dir, prune_dir, tmp_dir, output_base_template, valid_range, ) # Prune the MET .stat files and write to new file for met_stat_file in met_stat_files: - ps = subprocess.Popen('grep -R "'+model+'" '+met_stat_file+filter_cmd, - shell=True, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, encoding='UTF-8') - grep_output = ps.communicate()[0] + grep = subprocess.run('grep -R "'+model+'" '+met_stat_file+filter_cmd, + shell=True, capture_output=True, encoding="utf8") + grep_output = grep.stdout + all_grep_output = all_grep_output+grep_output pruned_met_stat_file = os.path.join(pruned_data_dir, model+'.stat') From 9e34f4969c18a67166b12027f4b92d5741438516 Mon Sep 17 00:00:00 2001 From: Samira Ardani <131818522+SamiraArdani-NOAA@users.noreply.github.com> Date: Fri, 22 Nov 2024 11:27:23 -0600 Subject: [PATCH 5/6] feature/EVS-NWPS_regions (#609) * Defined the region in driver script. * Generalized the Point-Stat and aggregated stats to include more WFOs. * Added changes to generalize plots to include more WFOs. * Defined WFO. * Added more changes to plots names. * WFO was added to the region in image names. * Added a note for file existence. * Added a minor change. * Minor change was added to a wfo name. * Added flexibility to codes in case of missing data from a wfo. * Modified the scripts to include all WFOs in one run. * Removed extra lines. * Removed extra driver scripts. * Removed a commented line. * Combined the do-loops into one. * Included all WFOs by running one driver script. * Added an if-statement in case of inapropriate data. * Added an if-statement in case of inapropriate data. * Added more changes. --- .../nwps/jevs_nwps_wave_grid2obs_plots.sh | 1 - .../nwps/jevs_nwps_wave_grid2obs_stats.sh | 0 jobs/JEVS_NWPS_PLOTS | 2 + jobs/JEVS_NWPS_STATS | 2 + .../wave_grid2obs/py_plotting_wave.config | 4 +- ...WPS_obsNDBC_climoERA5_Wave_Multifield.conf | 4 +- .../StatAnalysis_fcstNWPS_obsNDBC.conf | 6 +- .../nwps/exevs_nwps_wave_grid2obs_plots.sh | 212 +++++++++--------- .../nwps/exevs_nwps_wave_grid2obs_prep.sh | 2 +- .../nwps/exevs_nwps_wave_grid2obs_stats.sh | 104 +++++---- ush/nwps/evs_wave_leadaverages.sh | 56 ++--- ush/nwps/evs_wave_timeseries.sh | 127 +++++------ ush/nwps/lead_average.py | 40 ++-- ush/nwps/time_series.py | 34 +-- 14 files changed, 316 insertions(+), 278 deletions(-) mode change 100644 => 100755 dev/drivers/scripts/stats/nwps/jevs_nwps_wave_grid2obs_stats.sh diff --git a/dev/drivers/scripts/plots/nwps/jevs_nwps_wave_grid2obs_plots.sh b/dev/drivers/scripts/plots/nwps/jevs_nwps_wave_grid2obs_plots.sh index 265d70a50..288f48df2 100755 --- a/dev/drivers/scripts/plots/nwps/jevs_nwps_wave_grid2obs_plots.sh +++ b/dev/drivers/scripts/plots/nwps/jevs_nwps_wave_grid2obs_plots.sh @@ -13,7 +13,6 @@ export HOMEevs=/lfs/h2/emc/vpppg/noscrub/$USER/EVS export MODELNAME=nwps export OBTYPE=NDBC_STANDARD -export MASK=GOM export NET=evs export COMPONENT=nwps export STEP=plots diff --git a/dev/drivers/scripts/stats/nwps/jevs_nwps_wave_grid2obs_stats.sh b/dev/drivers/scripts/stats/nwps/jevs_nwps_wave_grid2obs_stats.sh old mode 100644 new mode 100755 diff --git a/jobs/JEVS_NWPS_PLOTS b/jobs/JEVS_NWPS_PLOTS index ca59df576..608fc623c 100755 --- a/jobs/JEVS_NWPS_PLOTS +++ b/jobs/JEVS_NWPS_PLOTS @@ -59,6 +59,7 @@ export COMIN=${COMIN:-$(compath.py ${envir}/com/${NET}/${evs_ver})} export COMOUT=${COMOUT:-$(compath.py -o ${NET}/${evs_ver})} export EVSINnwps=${COMIN}/stats/${COMPONENT}/${MODELNAME}.${VDATE} export COMOUTplots=${COMOUT}/${STEP}/${COMPONENT}/${RUN}.${VDATE} +export COMINnwps=${COMINnwps:-$(compath.py ${envir}/com/${MODELNAME}/${nwps_ver})} # make the directory where the plots will be kept mkdir -p ${COMOUTplots} @@ -67,6 +68,7 @@ mkdir -p ${COMOUTplots} #################################### # Execute the script #################################### + $HOMEevs/scripts/${STEP}/${COMPONENT}/exevs_${COMPONENT}_${RUN}_${VERIF_CASE}_${STEP}.sh export err=$?; err_chk diff --git a/jobs/JEVS_NWPS_STATS b/jobs/JEVS_NWPS_STATS index e6effc186..3c2ee4dd0 100755 --- a/jobs/JEVS_NWPS_STATS +++ b/jobs/JEVS_NWPS_STATS @@ -71,8 +71,10 @@ mkdir -p ${COMOUT} ${COMOUTsmall} ${COMOUTfinal} #################################### # Execute the script #################################### + $HOMEevs/scripts/${STEP}/${COMPONENT}/exevs_${COMPONENT}_${RUN}_${VERIF_CASE}_${STEP}.sh export err=$?; err_chk + msg="JOB $job HAS COMPLETED NORMALLY." diff --git a/parm/metplus_config/plots/nwps/wave_grid2obs/py_plotting_wave.config b/parm/metplus_config/plots/nwps/wave_grid2obs/py_plotting_wave.config index f8b045cf7..0a9c91470 100755 --- a/parm/metplus_config/plots/nwps/wave_grid2obs/py_plotting_wave.config +++ b/parm/metplus_config/plots/nwps/wave_grid2obs/py_plotting_wave.config @@ -136,9 +136,9 @@ export FCST_VALID_HOUR=${VHR} export FCST_LEVEL="L0" export OBS_LEVEL=${OBS_LEVEL} export var_name=${WVAR} -export VX_MASK_LIST=${MASK} +export VX_MASK_LIST="GLOBAL" export OBTYPE=${OBTYPE} - +export WFO=${wfo} # Will use statistics for all comma-separated lead times export FCST_LEAD=${FHR} diff --git a/parm/metplus_config/stats/nwps/wave_grid2obs/PointStat_fcstNWPS_obsNDBC_climoERA5_Wave_Multifield.conf b/parm/metplus_config/stats/nwps/wave_grid2obs/PointStat_fcstNWPS_obsNDBC_climoERA5_Wave_Multifield.conf index af10bcfea..833e06ce7 100644 --- a/parm/metplus_config/stats/nwps/wave_grid2obs/PointStat_fcstNWPS_obsNDBC_climoERA5_Wave_Multifield.conf +++ b/parm/metplus_config/stats/nwps/wave_grid2obs/PointStat_fcstNWPS_obsNDBC_climoERA5_Wave_Multifield.conf @@ -23,7 +23,7 @@ LEAD_SEQ = {ENV[lead]} ######################### FCST_POINT_STAT_INPUT_DIR = {ENV[DATA]}/gribs -FCST_POINT_STAT_INPUT_TEMPLATE = hgx_nwps_CG1.{init?fmt=%Y%m%d}.t{init?fmt=%2H}z.f{lead?fmt=%3H}.grib2 +FCST_POINT_STAT_INPUT_TEMPLATE = {ENV[wfo]}_nwps_CG1.{init?fmt=%Y%m%d}.t{init?fmt=%2H}z.f{lead?fmt=%3H}.grib2 FCST_POINT_STAT_INPUT_DATATYPE = GRIB2 FCST_IS_PROB = False @@ -41,7 +41,7 @@ OUTPUT_BASE = {ENV[DATA]} LOG_DIR = {OUTPUT_BASE}/logs TMP_DIR = {OUTPUT_BASE}/tmp -POINT_STAT_OUTPUT_DIR = {OUTPUT_BASE}/all_stats +POINT_STAT_OUTPUT_DIR = {OUTPUT_BASE}/all_stats/{ENV[wfo]} POINT_STAT_OUTPUT_TEMPLATE = POINT_STAT_OUTPUT_PREFIX = fcst{ENV[MODNAM]}_obsNDBC_climoERA5 diff --git a/parm/metplus_config/stats/nwps/wave_grid2obs/StatAnalysis_fcstNWPS_obsNDBC.conf b/parm/metplus_config/stats/nwps/wave_grid2obs/StatAnalysis_fcstNWPS_obsNDBC.conf index ac3955298..2850a3fe6 100644 --- a/parm/metplus_config/stats/nwps/wave_grid2obs/StatAnalysis_fcstNWPS_obsNDBC.conf +++ b/parm/metplus_config/stats/nwps/wave_grid2obs/StatAnalysis_fcstNWPS_obsNDBC.conf @@ -2,15 +2,17 @@ [config] #### Input -INPUT_BASE = {ENV[DATA]}/all_stats +INPUT_BASE = {ENV[DATA]}/all_stats/{ENV[wfo]} MODEL1_STAT_ANALYSIS_LOOKIN_DIR = {INPUT_BASE} #### Output OUTPUT_BASE = {ENV[DATA]} LOG_DIR = {OUTPUT_BASE}/logs TMP_DIR = {OUTPUT_BASE}/tmp -STAT_ANALYSIS_OUTPUT_DIR = {OUTPUT_BASE}/stats +STAT_ANALYSIS_OUTPUT_DIR = {OUTPUT_BASE}/stats/{ENV[wfo]} + ## this is the output file! + MODEL1_STAT_ANALYSIS_DUMP_ROW_TEMPLATE = evs.stats.{ENV[MODELNAME]}.{ENV[RUN]}.{ENV[VERIF_CASE]}.v{ENV[VDATE]}.stat LOG_METPLUS = {LOG_DIR}/StatAnalysis_fcst{MODEL1}_{ENV[VERIF_CASE]}_{ENV[STEP]}_for{VALID_BEG}.log diff --git a/scripts/plots/nwps/exevs_nwps_wave_grid2obs_plots.sh b/scripts/plots/nwps/exevs_nwps_wave_grid2obs_plots.sh index c64a9c9d0..d54ec4214 100755 --- a/scripts/plots/nwps/exevs_nwps_wave_grid2obs_plots.sh +++ b/scripts/plots/nwps/exevs_nwps_wave_grid2obs_plots.sh @@ -56,134 +56,134 @@ echo '-----------------------------' mkdir -p ${DATA}/stats -plot_start_date=${PDYm90} -plot_end_date=${VDATE} - -theDate=${plot_start_date} -while (( ${theDate} <= ${plot_end_date} )); do - EVSINnwps=${COMIN}/stats/${COMPONENT}/${MODELNAME}.${theDate} - if [ -s ${EVSINnwps}/evs.stats.${MODELNAME}.${RUN}.${VERIF_CASE}.v${theDate}.stat ]; then - cp ${EVSINnwps}/evs.stats.${MODELNAME}.${RUN}.${VERIF_CASE}.v${theDate}.stat ${DATA}/stats/. - else - echo "WARNING: ${EVSINnwps}/evs.stats.${MODELNAME}.${RUN}.${VERIF_CASE}.v${theDate}.stat DOES NOT EXIST" - fi - theDate=$(date --date="${theDate} + 1 day" '+%Y%m%d') -done +WFO='hgx bro' + +for wfo in ${WFO}; do + export wfo=$wfo + plot_start_date=${PDYm90} + plot_end_date=${VDATE} + + theDate=${plot_start_date} + while (( ${theDate} <= ${plot_end_date} )); do + EVSINnwps=${COMIN}/stats/${COMPONENT}/${MODELNAME}.${theDate} + if [ -s ${EVSINnwps}/evs.stats.${MODELNAME}.${wfo}.${RUN}.${VERIF_CASE}.v${theDate}.stat ]; then + cp ${EVSINnwps}/evs.stats.${MODELNAME}.${wfo}.${RUN}.${VERIF_CASE}.v${theDate}.stat ${DATA}/stats/. + else + echo "WARNING: ${EVSINnwps}/evs.stats.${MODELNAME}.${wfo}.${RUN}.${VERIF_CASE}.v${theDate}.stat DOES NOT EXIST" + fi + theDate=$(date --date="${theDate} + 1 day" '+%Y%m%d') + done #################### # quick error check #################### - -nc=`ls ${DATA}/stats/evs*stat | wc -l | awk '{print $1}'` -echo " Found ${nc} ${DATA}/stats/evs*stat file for ${VDATE} " -if [ "${nc}" != '0' ] -then - set -x - echo "Successfully copied the NWPS *.stat file for ${VDATE}" - [[ "$LOUD" = YES ]] && set -x -else - set -x - echo ' ' - echo '**************************************** ' - echo '*** FATAL ERROR: NO NWPS *.stat FILES *** ' - echo " for ${VDATE} " - echo '**************************************** ' - echo ' ' - echo "${MODELNAME}_${RUN} $VDATE $vhour : NWPS *.stat files missing." - [[ "$LOUD" = YES ]] && set -x - "FATAL ERROR: NO NWPS *.stat files for ${VDATE}" - err_exit "FATAL ERROR: Did not copy the NWPS *.stat files for ${VDATE}" - exit -fi - + nc=`ls ${DATA}/stats/evs.stats.nwps.${wfo}*stat | wc -l | awk '{print $1}'` + echo " Found ${nc} ${DATA}/stats/evs.stats.nwps.${wfo}*stat file for ${VDATE} " + if [ "${nc}" != '0' ] + then + set -x + echo "Successfully copied the NWPS *.stat file for ${VDATE}" + [[ "$LOUD" = YES ]] && set -x + else + set -x + echo ' ' + echo '**************************************** ' + echo '*** FATAL ERROR: NO NWPS *.stat FILES *** ' + echo " for ${wfo} at ${VDATE} " + echo '**************************************** ' + echo ' ' + echo "${MODELNAME}_${RUN} $VDATE $vhour : NWPS *.stat files missing." + [[ "$LOUD" = YES ]] && set -x + "FATAL ERROR: NO NWPS *.stat files for ${VDATE}" + err_exit "FATAL ERROR: Did not copy the NWPS *.stat files for ${VDATE}" + exit + fi ################################# ## Make the command files for cfp ################################# -# time_series -${USHevs}/${COMPONENT}/evs_wave_timeseries.sh -export err=$?; err_chk - -# lead_averages -${USHevs}/${COMPONENT}/evs_wave_leadaverages.sh -export err=$?; err_chk + # time_series + ${USHevs}/${COMPONENT}/evs_wave_timeseries.sh + export err=$?; err_chk -chmod 775 plot_all_${MODELNAME}_${RUN}_g2o_plots.sh + # lead_averages + ${USHevs}/${COMPONENT}/evs_wave_leadaverages.sh + export err=$?; err_chk ########################################### # Run the command files for the PAST31DAYS ########################################### - -if [ ${run_mpi} = 'yes' ] ; then - mpiexec -np 36 --cpu-bind verbose,core cfp plot_all_${MODELNAME}_${RUN}_g2o_plots.sh -else - echo "not running mpiexec" - sh plot_all_${MODELNAME}_${RUN}_g2o_plots.sh -fi + cd ${DATA} + chmod 775 ${DATA}/plot_all_${MODELNAME}_${RUN}_g2o_${wfo}_plots.sh + if [ ${run_mpi} = 'yes' ] ; then + mpiexec -np 36 --cpu-bind verbose,core cfp plot_all_${MODELNAME}_${RUN}_g2o_${wfo}_plots.sh + else + echo "not running mpiexec" + sh plot_all_${MODELNAME}_${RUN}_g2o_${wfo}_plots.sh + fi ##################### # Gather all the files ####################### -periods='LAST31DAYS LAST90DAYS' -if [ $gather = yes ] ; then - echo "copying all images into one directory" - cp ${DATA}/wave/*png ${DATA}/ndbc_standard/. ## lead_average plots - nc=$(ls ${DATA}/ndbc_standard/*.fhrmean_valid*.png | wc -l | awk '{print $1}') - echo "copied $nc lead_average plots" - for period in ${periods} ; do - period_lower=$(echo ${period,,}) - if [ ${period} = 'LAST31DAYS' ] ; then - period_out='last31days' - elif [ ${period} = 'LAST90DAYS' ] ; then - period_out='last90days' - fi - - # check to see if the plots are there - nc=$(ls ${DATA}/ndbc_standard/*${period_lower}*.png | wc -l | awk '{print $1}') - echo " Found ${nc} ${DATA}/plots/*${period_lower}*.png files for ${VDATE} " - if [ "${nc}" != '0' ] - then - set -x - echo "Found ${nc} ${period_lower} plots for ${VDATE}" - [[ "$LOUD" = YES ]] && set -x - else - set -x - echo ' ' - echo '**************************************** ' - echo "*** FATAL ERROR: NO ${period} PLOTS *** " - echo " found for ${VDATE} " - echo '**************************************** ' - echo ' ' - echo "${MODELNAME}_${RUN} ${VDATE} ${period}: PLOTS are missing." - [[ "$LOUD" = YES ]] && set -x - - "FATAL ERROR : NO ${period} PLOTS FOR ${VDATE}" - err_exit "FATAL ERROR: Did not find any ${period} plots for ${VDATE}" - fi - # tar and copy them to the final destination - - if [ "${nc}" > '0' ] ; then - cd ${DATA}/ndbc_standard - tar -cvf evs.${STEP}.${COMPONENT}.${RUN}.${VERIF_CASE}.${period_out}.v${VDATE}.tar evs.*${period_lower}*.png - fi - if [ $SENDCOM = YES ]; then - if [ -s evs.${STEP}.${COMPONENT}.${RUN}.${VERIF_CASE}.${period_out}.v${VDATE}.tar ]; then - cp -v evs.${STEP}.${COMPONENT}.${RUN}.${VERIF_CASE}.${period_out}.v${VDATE}.tar ${COMOUTplots} + periods='LAST31DAYS LAST90DAYS' + if [ $gather = yes ] ; then + echo "copying all images into one directory" + cp ${DATA}/wave/*png ${DATA}/ndbc_standard/. ## lead_average plots + nc=$(ls ${DATA}/ndbc_standard/*.fhrmean_valid*.png | wc -l | awk '{print $1}') + echo "copied $nc lead_average plots" + for period in ${periods} ; do + period_lower=$(echo ${period,,}) + if [ ${period} = 'LAST31DAYS' ] ; then + period_out='last31days' + elif [ ${period} = 'LAST90DAYS' ] ; then + period_out='last90days' fi - fi - if [ $SENDDBN = YES ]; then - $DBNROOT/bin/dbn_alert MODEL EVS_RZDM $job ${COMOUTplots}/${NET}.${STEP}.${COMPONENT}.${RUN}.${VERIF_CASE}.${period_out}.v${VDATE}.tar - fi - done - -else - echo "not copying the plots" -fi + # check to see if the plots are there + nc=$(ls ${DATA}/ndbc_standard/*${period_lower}*.png | wc -l | awk '{print $1}') + echo " Found ${nc} ${DATA}/plots/*${period_lower}*.png files for ${VDATE} " + if [ "${nc}" != '0' ] + then + set -x + echo "Found ${nc} ${period_lower} plots for ${VDATE}" + [[ "$LOUD" = YES ]] && set -x + else + set -x + echo ' ' + echo '**************************************** ' + echo "*** FATAL ERROR: NO ${period} PLOTS *** " + echo " found for ${VDATE} " + echo '**************************************** ' + echo ' ' + echo "${MODELNAME}_${RUN} ${VDATE} ${period}: PLOTS are missing." + [[ "$LOUD" = YES ]] && set -x + + "FATAL ERROR : NO ${period} PLOTS FOR ${VDATE}" + err_exit "FATAL ERROR: Did not find any ${period} plots for ${VDATE}" + fi + # tar and copy them to the final destination + + if [ "${nc}" > '0' ] ; then + cd ${DATA}/ndbc_standard + tar -cvf evs.${STEP}.${COMPONENT}.${wfo}.${RUN}.${VERIF_CASE}.${period_out}.v${VDATE}.tar evs.*${period_lower}*_${wfo}.png + fi + if [ $SENDCOM = YES ]; then + if [ -s evs.${STEP}.${COMPONENT}.${wfo}.${RUN}.${VERIF_CASE}.${period_out}.v${VDATE}.tar ]; then + cp -v evs.${STEP}.${COMPONENT}.${wfo}.${RUN}.${VERIF_CASE}.${period_out}.v${VDATE}.tar ${COMOUTplots}/evs.${STEP}.${COMPONENT}.${wfo}.${RUN}.${VERIF_CASE}.${period_out}.v${VDATE}.tar + fi + fi + if [ $SENDDBN = YES ]; then + $DBNROOT/bin/dbn_alert MODEL EVS_RZDM $job ${COMOUTplots}/${NET}.${STEP}.${COMPONENT}.${wfo}.${RUN}.${VERIF_CASE}.${period_out}.v${VDATE}.tar + fi -msg="JOB $job HAS COMPLETED NORMALLY." + done + else + echo "not copying the plots" + fi + msg="JOB $job HAS COMPLETED NORMALLY." +done ######################################### # copy log files into logs and cat them ######################################### diff --git a/scripts/prep/nwps/exevs_nwps_wave_grid2obs_prep.sh b/scripts/prep/nwps/exevs_nwps_wave_grid2obs_prep.sh index 2176e1893..76940682f 100755 --- a/scripts/prep/nwps/exevs_nwps_wave_grid2obs_prep.sh +++ b/scripts/prep/nwps/exevs_nwps_wave_grid2obs_prep.sh @@ -52,7 +52,7 @@ for region in ${regions} ; do fi done -wfos='aer afg ajk alu akq box car chs gys olm lwx mhx okx phi gum hfo bro crp hgx jax key lch lix mfi mlb mob sju tae tbw eka lox mfr mtr pqr sew sgx' +wfos='aer afg ajk alu akq box car chs gys olm lwx mhx okx phi gum hfo bro crp hgx jax key lch lix mfl mlb mob sju tae tbw eka lox mfr mtr pqr sew sgx' #CG1 is the main domain. CG2-CG6 are the nested domains. #CGs='CG1 CG2 CG3 CG4 CG5 CG6' CGs='CG1' diff --git a/scripts/stats/nwps/exevs_nwps_wave_grid2obs_stats.sh b/scripts/stats/nwps/exevs_nwps_wave_grid2obs_stats.sh index f2a9cce5b..21650de1d 100755 --- a/scripts/stats/nwps/exevs_nwps_wave_grid2obs_stats.sh +++ b/scripts/stats/nwps/exevs_nwps_wave_grid2obs_stats.sh @@ -39,10 +39,11 @@ mkdir -p ${DATA}/tmp mkdir -p ${DATA}/SFCSHP vhours='00 06 12 18' -WFO='hgx' +WFO='hgx bro' CG='CG1' lead_hours='0 24 48 72 96 120 144' + export GRID2OBS_CONF="${PARMevs}/metplus_config/${STEP}/${COMPONENT}/${RUN}_${VERIF_CASE}" cd ${DATA} @@ -54,6 +55,12 @@ echo ' ' echo 'Creating point_stat files' for wfo in ${WFO}; do + export wfo=$wfo + mkdir -p ${DATA}/jobs/${wfo} + mkdir -p $COMOUTsmall/${wfo} + mkdir -p ${DATA}/all_stats/${wfo} + mkdir -p ${DATA}/stats + mkdir -p ${DATA}/stats/${wfo} for cg in ${CG}; do for vhr in ${vhours} ; do vhr2=$(printf "%02d" "${vhr}") @@ -95,8 +102,8 @@ for wfo in ${WFO}; do DATAobsfilename=${DATA}/ncfiles/ndbc.${VDATE}.nc done - DATAstatfilename=$DATA/all_stats/point_stat_fcst${MODNAM}_obs${OBSNAME}_climoERA5_${flead2}0000L_${VDATE}_${vhr2}0000V.stat - COMOUTstatfilename=$COMOUTsmall/point_stat_fcst${MODNAM}_obs${OBSNAME}_climoERA5_${flead2}0000L_${VDATE}_${vhr2}0000V.stat + DATAstatfilename=$DATA/all_stats/${wfo}/point_stat_fcst${MODNAM}_obs${OBSNAME}_climoERA5_${flead2}0000L_${VDATE}_${vhr2}0000V.stat + COMOUTstatfilename=$COMOUTsmall/${wfo}/point_stat_fcst${MODNAM}_obs${OBSNAME}_climoERA5_${flead2}0000L_${VDATE}_${vhr2}0000V.stat if [[ -s $COMOUTstatfilename ]]; then echo "RESTART: Copy the files" @@ -114,70 +121,73 @@ for wfo in ${WFO}; do if [[ -s $EVSINmodelfilename ]]; then cp -v $EVSINmodelfilename $DATAmodelfilename else - echo "DOES NOT EXIST $EVSINobsfilename" + echo "DOES NOT EXIST $EVSINmodelfilename" fi fi if [[ -s $DATAmodelfilename ]]; then - echo "export wind_level_str=${wind_level_str}" >> ${DATA}/jobs/run_${MODELNAME}_${RUN}_${VDATE}${vhr2}_f${flead}_g2o.sh - echo "export htsgw_level_str=${htsgw_level_str}" >> ${DATA}/jobs/run_${MODELNAME}_${RUN}_${VDATE}${vhr2}_f${flead}_g2o.sh - echo "export perpw_level_str=${perpw_level_str}" >> ${DATA}/jobs/run_${MODELNAME}_${RUN}_${VDATE}${vhr2}_f${flead}_g2o.sh - echo "export wdir_level_str=${wdir_level_str}" >> ${DATA}/jobs/run_${MODELNAME}_${RUN}_${VDATE}${vhr2}_f${flead}_g2o.sh - echo "export VHR=${vhr2}" >> ${DATA}/jobs/run_${MODELNAME}_${RUN}_${VDATE}${vhr2}_f${flead}_g2o.sh - echo "export lead=${flead}" >> ${DATA}/jobs/run_${MODELNAME}_${RUN}_${VDATE}${vhr2}_f${flead}_g2o.sh - echo "${METPLUS_PATH}/ush/run_metplus.py ${PARMevs}/metplus_config/machine.conf ${GRID2OBS_CONF}/PointStat_fcstNWPS_obs${OBSNAME}_climoERA5_Wave_Multifield.conf" >> ${DATA}/jobs/run_${MODELNAME}_${RUN}_${VDATE}${vhr2}_f${flead}_g2o.sh - echo "export err=\$?; err_chk" >> ${DATA}/jobs/run_${MODELNAME}_${RUN}_${VDATE}${vhr2}_f${flead}_g2o.sh + echo "export wind_level_str=${wind_level_str}" >> ${DATA}/jobs/${wfo}/run_${MODELNAME}_${RUN}_${VDATE}${vhr2}_f${flead}_g2o.sh + echo "export htsgw_level_str=${htsgw_level_str}" >> ${DATA}/jobs/${wfo}/run_${MODELNAME}_${RUN}_${VDATE}${vhr2}_f${flead}_g2o.sh + echo "export perpw_level_str=${perpw_level_str}" >> ${DATA}/jobs/${wfo}/run_${MODELNAME}_${RUN}_${VDATE}${vhr2}_f${flead}_g2o.sh + echo "export wdir_level_str=${wdir_level_str}" >> ${DATA}/jobs/${wfo}/run_${MODELNAME}_${RUN}_${VDATE}${vhr2}_f${flead}_g2o.sh + echo "export VHR=${vhr2}" >> ${DATA}/jobs/${wfo}/run_${MODELNAME}_${RUN}_${VDATE}${vhr2}_f${flead}_g2o.sh + echo "export lead=${flead}" >> ${DATA}/jobs/${wfo}/run_${MODELNAME}_${RUN}_${VDATE}${vhr2}_f${flead}_g2o.sh + echo "export wfo=${wfo}" >> ${DATA}/jobs/${wfo}/run_${MODELNAME}_${RUN}_${VDATE}${vhr2}_f${flead}_g2o.sh + echo "${METPLUS_PATH}/ush/run_metplus.py ${PARMevs}/metplus_config/machine.conf ${GRID2OBS_CONF}/PointStat_fcstNWPS_obs${OBSNAME}_climoERA5_Wave_Multifield.conf" >> ${DATA}/jobs/${wfo}/run_${MODELNAME}_${RUN}_${VDATE}${vhr2}_f${flead}_g2o.sh + echo "export err=\$?; err_chk" >> ${DATA}/jobs/${wfo}/run_${MODELNAME}_${RUN}_${VDATE}${vhr2}_f${flead}_g2o.sh if [ $SENDCOM = YES ]; then - echo "if [ -s $DATAstatfilename ]; then cp -v $DATAstatfilename $COMOUTstatfilename; fi" >> ${DATA}/jobs/run_${MODELNAME}_${RUN}_${VDATE}${vhr2}_f${flead}_g2o.sh + echo "if [ -s $DATAstatfilename ]; then cp -v $DATAstatfilename $COMOUTstatfilename; fi" >> ${DATA}/jobs/${wfo}/run_${MODELNAME}_${RUN}_${VDATE}${vhr2}_f${flead}_g2o.sh fi - chmod +x ${DATA}/jobs/run_${MODELNAME}_${RUN}_${VDATE}${vhr2}_f${flead}_g2o.sh - echo "${DATA}/jobs/run_${MODELNAME}_${RUN}_${VDATE}${vhr2}_f${flead}_g2o.sh" >> ${DATA}/jobs/run_all_${MODELNAME}_${RUN}_g2o_poe.sh + chmod +x ${DATA}/jobs/${wfo}/run_${MODELNAME}_${RUN}_${VDATE}${vhr2}_f${flead}_g2o.sh + echo "${DATA}/jobs/${wfo}/run_${MODELNAME}_${RUN}_${VDATE}${vhr2}_f${flead}_g2o.sh" >> ${DATA}/jobs/${wfo}/run_all_${MODELNAME}_${RUN}_g2o_poe.sh + else + echo "DOES NOT EXIST $DATAmodelfilename" fi fi fi done done done -done ######################### # Run the command file ######################### -if [[ -s ${DATA}/jobs/run_all_${MODELNAME}_${RUN}_g2o_poe.sh ]]; then - if [ ${run_mpi} = 'yes' ] ; then - export LD_LIBRARY_PATH=/apps/dev/pmi-fix:$LD_LIBRARY_PATH - mpiexec -np 36 -ppn 36 --cpu-bind verbose,core cfp ${DATA}/jobs/run_all_${MODELNAME}_${RUN}_g2o_poe.sh - export err=$?; err_chk - - else - echo "not running mpiexec" - sh ${DATA}/jobs/run_all_${MODELNAME}_${RUN}_g2o_poe.sh - fi -fi + if [[ -s ${DATA}/jobs/${wfo}/run_all_${MODELNAME}_${RUN}_g2o_poe.sh ]]; then + if [ ${run_mpi} = 'yes' ] ; then + export LD_LIBRARY_PATH=/apps/dev/pmi-fix:$LD_LIBRARY_PATH + mpiexec -np 36 -ppn 36 --cpu-bind verbose,core cfp ${DATA}/jobs/${wfo}/run_all_${MODELNAME}_${RUN}_g2o_poe.sh + export err=$?; err_chk + + else + echo "not running mpiexec" + sh ${DATA}/jobs/${wfo}/run_all_${MODELNAME}_${RUN}_g2o_poe.sh + fi + fi ########################## # Gather all the files ######################### -if [ $gather = yes ] ; then -# check to see if the small stat files are there - nc=$(ls ${DATA}/all_stats/*stat | wc -l | awk '{print $1}') - if [ "${nc}" != '0' ]; then - echo " Found ${nc} ${DATA}/all_stats/*stat files for ${VDATE}" - mkdir -p ${DATA}/stats - # Use StatAnalysis to gather the small stat files into one file - run_metplus.py ${PARMevs}/metplus_config/machine.conf ${GRID2OBS_CONF}/StatAnalysis_fcstNWPS_obs$OBSNAME.conf - export err=$?; err_chk - - if [ $SENDCOM = YES ]; then - if [ -s ${DATA}/stats/evs.stats.${MODELNAME}.${RUN}.${VERIF_CASE}.v${VDATE}.stat ]; then - cp -v ${DATA}/stats/evs.stats.${MODELNAME}.${RUN}.${VERIF_CASE}.v${VDATE}.stat ${COMOUTfinal}/. - else - echo "DOES NOT EXIST ${DATA}/stats/evs.stats.${MODELNAME}.${RUN}.${VERIF_CASE}.v${VDATE}.stat" + if [ $gather = yes ] ; then + # check to see if the small stat files are there + #mkdir -p ${DATA}/stats + #mkdir -p ${DATA}/stats/${wfo} + nc=$(ls ${DATA}/all_stats/${wfo}/*stat | wc -l | awk '{print $1}') + if [ "${nc}" != '0' ]; then + echo " Found ${nc} ${DATA}/all_stats/${wfo}/*stat files for ${VDATE}" + # Use StatAnalysis to gather the small stat files into one file + run_metplus.py ${PARMevs}/metplus_config/machine.conf ${GRID2OBS_CONF}/StatAnalysis_fcstNWPS_obs$OBSNAME.conf + export err=$?; err_chk + + if [ $SENDCOM = YES ]; then + if [ -s ${DATA}/stats/${wfo}/evs.stats.${MODELNAME}.${RUN}.${VERIF_CASE}.v${VDATE}.stat ]; then + cp -v ${DATA}/stats/${wfo}/evs.stats.${MODELNAME}.${RUN}.${VERIF_CASE}.v${VDATE}.stat ${COMOUTfinal}/evs.stats.${MODELNAME}.${wfo}.${RUN}.${VERIF_CASE}.v${VDATE}.stat + else + echo "DOES NOT EXIST ${DATA}/stats/${wfo}/evs.stats.${MODELNAME}.${RUN}.${VERIF_CASE}.v${VDATE}.stat" + fi fi + else + echo "NO SMALL STAT FILES FOUND IN ${DATA}/all_stats/${wfo}" fi - else - echo "NO SMALL STAT FILES FOUND IN ${DATA}/all_stats" - fi -fi - + fi +done ############################################################################### echo ' ' echo "Ending at : `date`" diff --git a/ush/nwps/evs_wave_leadaverages.sh b/ush/nwps/evs_wave_leadaverages.sh index e8ff2ac2e..fe2de4e1e 100755 --- a/ush/nwps/evs_wave_leadaverages.sh +++ b/ush/nwps/evs_wave_leadaverages.sh @@ -23,7 +23,8 @@ ptype='lead_average' export GRID2OBS_CONF="${PARMevs}/metplus_config/${STEP}/${COMPONENT}/${RUN}_${VERIF_CASE}" cd ${DATA} -touch plot_all_${MODELNAME}_${RUN}_g2o_plots.sh +export wfo=$wfo +touch plot_all_${MODELNAME}_${RUN}_g2o_${wfo}_plots.sh # write the commands for period in ${periods} ; do @@ -35,54 +36,55 @@ for period in ${periods} ; do for vhr in ${inithours} ; do for wvar in ${wave_vars} ; do for stats in ${stats_list}; do - echo "export VX_MASK_LIST=${MASK} " >> plot_${wvar}_${vhr}_${fhr}_${stats}_${ptype}_${period}.sh - echo "export VERIF_CASE=${VERIF_CASE} " >> plot_${wvar}_${vhr}_${stats}_${ptype}_${period}.sh - echo "export RUN=${RUN} " >> plot_${wvar}_${vhr}_${stats}_${ptype}_${period}.sh - echo "export USHevs=${USHevs}/${COMPONENT} " >> plot_${wvar}_${vhr}_${stats}_${ptype}_${period}.sh - echo "export FIXevs=${FIXevs} " >> plot_${wvar}_${vhr}_${stats}_${ptype}_${period}.sh - echo "export DATA=${DATA} " >> plot_${wvar}_${vhr}_${stats}_${ptype}_${period}.sh - echo "export MODNAM=${MODNAM} " >> plot_${wvar}_${vhr}_${stats}_${ptype}_${period}.sh - echo "export PERIOD=${period} " >> plot_${wvar}_${vhr}_${stats}_${ptype}_${period}.sh - echo "export VERIF_CASE=${VERIF_CASE} " >> plot_${wvar}_${vhr}_${stats}_${ptype}_${period}.sh - echo "export plot_start_date=${plot_start_date} " >> plot_${wvar}_${vhr}_${stats}_${ptype}_${period}.sh - echo "export plot_end_date=${VDATE} " >> plot_${wvar}_${vhr}_${stats}_${ptype}_${period}.sh - echo "export VHR=${vhr} " >> plot_${wvar}_${vhr}_${stats}_${ptype}_${period}.sh + echo "export VX_MASK_LIST=${MASK} " >> plot_${wfo}_${wvar}_${vhr}_${stats}_${ptype}_${period}.sh + echo "export VERIF_CASE=${VERIF_CASE} " >> plot_${wfo}_${wvar}_${vhr}_${stats}_${ptype}_${period}.sh + echo "export RUN=${RUN} " >> plot_${wfo}_${wvar}_${vhr}_${stats}_${ptype}_${period}.sh + echo "export USHevs=${USHevs}/${COMPONENT} " >> plot_${wfo}_${wvar}_${vhr}_${stats}_${ptype}_${period}.sh + echo "export FIXevs=${FIXevs} " >> plot_${wfo}_${wvar}_${vhr}_${stats}_${ptype}_${period}.sh + echo "export DATA=${DATA} " >> plot_${wfo}_${wvar}_${vhr}_${stats}_${ptype}_${period}.sh + echo "export MODNAM=${MODNAM} " >> plot_${wfo}_${wvar}_${vhr}_${stats}_${ptype}_${period}.sh + echo "export PERIOD=${period} " >> plot_${wfo}_${wvar}_${vhr}_${stats}_${ptype}_${period}.sh + echo "export VERIF_CASE=${VERIF_CASE} " >> plot_${wfo}_${wvar}_${vhr}_${stats}_${ptype}_${period}.sh + echo "export plot_start_date=${plot_start_date} " >> plot_${wfo}_${wvar}_${vhr}_${stats}_${ptype}_${period}.sh + echo "export plot_end_date=${VDATE} " >> plot_${wfo}_${wvar}_${vhr}_${stats}_${ptype}_${period}.sh + echo "export VHR=${vhr} " >> plot_${wfo}_${wvar}_${vhr}_${stats}_${ptype}_${period}.sh + echo "export WFO=${wfo} " >> plot_${wfo}_${wvar}_${vhr}_${stats}_${ptype}_${period}.sh case ${stats} in 'stats1') - echo "export METRIC='me, rmse' " >> plot_${wvar}_${vhr}_${stats}_${ptype}_${period}.sh + echo "export METRIC='me, rmse' " >> plot_${wfo}_${wvar}_${vhr}_${stats}_${ptype}_${period}.sh ;; 'stats2') - echo "export METRIC=pcor " >> plot_${wvar}_${vhr}_${stats}_${ptype}_${period}.sh + echo "export METRIC=pcor " >> plot_${wfo}_${wvar}_${vhr}_${stats}_${ptype}_${period}.sh ;; 'stats3') - echo "export METRIC='fbar, obar' " >> plot_${wvar}_${vhr}_${stats}_${ptype}_${period}.sh + echo "export METRIC='fbar, obar' " >> plot_${wfo}_${wvar}_${vhr}_${stats}_${ptype}_${period}.sh ;; 'stats4') - echo "export METRIC=esd " >> plot_${wvar}_${vhr}_${stats}_${ptype}_${period}.sh + echo "export METRIC=esd " >> plot_${wfo}_${wvar}_${vhr}_${stats}_${ptype}_${period}.sh ;; 'stats5') - echo "export METRIC=si " >> plot_${wvar}_${vhr}_${stats}_${ptype}_${period}.sh + echo "export METRIC=si " >> plot_${wfo}_${wvar}_${vhr}_${stats}_${ptype}_${period}.sh ;; 'stats6') - echo "export METRIC=p95 " >> plot_${wvar}_${vhr}_${stats}_${ptype}_${period}.sh + echo "export METRIC=p95 " >> plot_${wfo}_${wvar}_${vhr}_${stats}_${ptype}_${period}.sh ;; esac - echo "export FHR='000,024,048,072,096,120,144,168,192,216,240'" >> plot_${wvar}_${vhr}_${stats}_${ptype}_${period}.sh - echo "export WVAR=${wvar}" >> plot_${wvar}_${vhr}_${stats}_${ptype}_${period}.sh + echo "export FHR='000,024,048,072,096,120,144,168,192,216,240'" >> plot_${wfo}_${wvar}_${vhr}_${stats}_${ptype}_${period}.sh + echo "export WVAR=${wvar}" >> plot_${wfo}_${wvar}_${vhr}_${stats}_${ptype}_${period}.sh case ${wvar} in 'WIND') - echo "export OBS_LEVEL=Z10" >> plot_${wvar}_${vhr}_${stats}_${ptype}_${period}.sh + echo "export OBS_LEVEL=Z10" >> plot_${wfo}_${wvar}_${vhr}_${stats}_${ptype}_${period}.sh ;; *) - echo "export OBS_LEVEL=L0" >> plot_${wvar}_${vhr}_${stats}_${ptype}_${period}.sh + echo "export OBS_LEVEL=L0" >> plot_${wfo}_${wvar}_${vhr}_${stats}_${ptype}_${period}.sh ;; esac - echo "export PTYPE=${ptype}" >> plot_${wvar}_${vhr}_${stats}_${ptype}_${period}.sh - echo "${GRID2OBS_CONF}/py_plotting_wave.config" >> plot_${wvar}_${vhr}_${stats}_${ptype}_${period}.sh + echo "export PTYPE=${ptype}" >> plot_${wfo}_${wvar}_${vhr}_${stats}_${ptype}_${period}.sh + echo "${GRID2OBS_CONF}/py_plotting_wave.config" >> plot_${wfo}_${wvar}_${vhr}_${stats}_${ptype}_${period}.sh - chmod +x plot_${wvar}_${vhr}_${stats}_${ptype}_${period}.sh + chmod +x plot_${wfo}_${wvar}_${vhr}_${stats}_${ptype}_${period}.sh - echo "${DATA}/plot_${wvar}_${vhr}_${stats}_${ptype}_${period}.sh" >> plot_all_${MODELNAME}_${RUN}_g2o_plots.sh + echo "${DATA}/plot_${wfo}_${wvar}_${vhr}_${stats}_${ptype}_${period}.sh" >> plot_all_${MODELNAME}_${RUN}_g2o_${wfo}_plots.sh done # end of stats done # end of wave vars diff --git a/ush/nwps/evs_wave_timeseries.sh b/ush/nwps/evs_wave_timeseries.sh index 015a70eba..7a227a611 100755 --- a/ush/nwps/evs_wave_timeseries.sh +++ b/ush/nwps/evs_wave_timeseries.sh @@ -20,73 +20,74 @@ stats_list='stats1 stats2 stats3 stats4 stats5' ptype='time_series' export GRID2OBS_CONF="${PARMevs}/metplus_config/${STEP}/${COMPONENT}/${RUN}_${VERIF_CASE}" - cd ${DATA} -touch plot_all_${MODELNAME}_${RUN}_g2o_plots.sh # write the commands + +export wfo=$wfo +touch plot_all_${MODELNAME}_${RUN}_g2o_${wfo}_plots.sh for period in ${periods} ; do - if [ ${period} = 'LAST31DAYS' ] ; then - plot_start_date=${PDYm31} - elif [ ${period} = 'LAST90DAYS' ] ; then - plot_start_date=${PDYm90} - fi - for vhr in ${inithours} ; do - for wvar in ${wave_vars} ; do - for stats in ${stats_list}; do - for fhr in ${fhrs} ; do - echo "export VX_MASK_LIST=${MASK} " >> plot_${wvar}_${vhr}_${fhr}_${stats}_${ptype}_${period}.sh - echo "export VERIF_CASE=${VERIF_CASE} " >> plot_${wvar}_${vhr}_${fhr}_${stats}_${ptype}_${period}.sh - echo "export RUN=${RUN} " >> plot_${wvar}_${vhr}_${fhr}_${stats}_${ptype}_${period}.sh - echo "export USHevs=${USHevs}/${COMPONENT} " >> plot_${wvar}_${vhr}_${fhr}_${stats}_${ptype}_${period}.sh - echo "export FIXevs=${FIXevs} " >> plot_${wvar}_${vhr}_${fhr}_${stats}_${ptype}_${period}.sh - echo "export DATA=${DATA} " >> plot_${wvar}_${vhr}_${fhr}_${stats}_${ptype}_${period}.sh - echo "export MODNAM=${MODNAM} " >> plot_${wvar}_${vhr}_${fhr}_${stats}_${ptype}_${period}.sh - echo "export PERIOD=${period} " >> plot_${wvar}_${vhr}_${fhr}_${stats}_${ptype}_${period}.sh - echo "export VERIF_CASE=${VERIF_CASE} " >> plot_${wvar}_${vhr}_${fhr}_${stats}_${ptype}_${period}.sh - echo "export plot_start_date=${plot_start_date} " >> plot_${wvar}_${vhr}_${fhr}_${stats}_${ptype}_${period}.sh - echo "export plot_end_date=${VDATE} " >> plot_${wvar}_${vhr}_${fhr}_${stats}_${ptype}_${period}.sh - echo "export VHR=${vhr} " >> plot_${wvar}_${vhr}_${fhr}_${stats}_${ptype}_${period}.sh - case ${stats} in - 'stats1') - echo "export METRIC='me, rmse' " >> plot_${wvar}_${vhr}_${fhr}_${stats}_${ptype}_${period}.sh - ;; - 'stats2') - echo "export METRIC=pcor " >> plot_${wvar}_${vhr}_${fhr}_${stats}_${ptype}_${period}.sh - ;; - 'stats3') - echo "export METRIC='fbar, obar' " >> plot_${wvar}_${vhr}_${fhr}_${stats}_${ptype}_${period}.sh - ;; - 'stats4') - echo "export METRIC=esd " >> plot_${wvar}_${vhr}_${fhr}_${stats}_${ptype}_${period}.sh - ;; - 'stats5') - echo "export METRIC=si " >> plot_${wvar}_${vhr}_${fhr}_${stats}_${ptype}_${period}.sh - ;; - 'stats6') - echo "export METRIC=p95 " >> plot_${wvar}_${vhr}_${fhr}_${stats}_${ptype}_${period}.sh - ;; - esac - echo "export FHR=${fhr}" >> plot_${wvar}_${vhr}_${fhr}_${stats}_${ptype}_${period}.sh - echo "export WVAR=${wvar}" >> plot_${wvar}_${vhr}_${fhr}_${stats}_${ptype}_${period}.sh - case ${wvar} in - 'WIND') - echo "export OBS_LEVEL=Z10" >> plot_${wvar}_${vhr}_${fhr}_${stats}_${ptype}_${period}.sh - ;; - *) - echo "export OBS_LEVEL=L0" >> plot_${wvar}_${vhr}_${fhr}_${stats}_${ptype}_${period}.sh - ;; - esac - echo "export PTYPE=${ptype}" >> plot_${wvar}_${vhr}_${fhr}_${stats}_${ptype}_${period}.sh - echo "${GRID2OBS_CONF}/py_plotting_wave.config" >> plot_${wvar}_${vhr}_${fhr}_${stats}_${ptype}_${period}.sh - - chmod +x plot_${wvar}_${vhr}_${fhr}_${stats}_${ptype}_${period}.sh + if [ ${period} = 'LAST31DAYS' ] ; then + plot_start_date=${PDYm31} + elif [ ${period} = 'LAST90DAYS' ] ; then + plot_start_date=${PDYm90} + fi + for vhr in ${inithours} ; do + for wvar in ${wave_vars} ; do + for stats in ${stats_list}; do + for fhr in ${fhrs} ; do + echo "export VX_MASK_LIST=${MASK} " >> plot_${wfo}_${wvar}_${vhr}_${fhr}_${stats}_${ptype}_${period}.sh + echo "export VERIF_CASE=${VERIF_CASE} " >> plot_${wfo}_${wvar}_${vhr}_${fhr}_${stats}_${ptype}_${period}.sh + echo "export RUN=${RUN} " >> plot_${wfo}_${wvar}_${vhr}_${fhr}_${stats}_${ptype}_${period}.sh + echo "export USHevs=${USHevs}/${COMPONENT} " >> plot_${wfo}_${wvar}_${vhr}_${fhr}_${stats}_${ptype}_${period}.sh + echo "export FIXevs=${FIXevs} " >> plot_${wfo}_${wvar}_${vhr}_${fhr}_${stats}_${ptype}_${period}.sh + echo "export DATA=${DATA} " >> plot_${wfo}_${wvar}_${vhr}_${fhr}_${stats}_${ptype}_${period}.sh + echo "export MODNAM=${MODNAM} " >> plot_${wfo}_${wvar}_${vhr}_${fhr}_${stats}_${ptype}_${period}.sh + echo "export PERIOD=${period} " >> plot_${wfo}_${wvar}_${vhr}_${fhr}_${stats}_${ptype}_${period}.sh + echo "export VERIF_CASE=${VERIF_CASE} " >> plot_${wfo}_${wvar}_${vhr}_${fhr}_${stats}_${ptype}_${period}.sh + echo "export plot_start_date=${plot_start_date} " >> plot_${wfo}_${wvar}_${vhr}_${fhr}_${stats}_${ptype}_${period}.sh + echo "export plot_end_date=${VDATE} " >> plot_${wfo}_${wvar}_${vhr}_${fhr}_${stats}_${ptype}_${period}.sh + echo "export VHR=${vhr} " >> plot_${wfo}_${wvar}_${vhr}_${fhr}_${stats}_${ptype}_${period}.sh + echo "export WFO=${wfo} " >> plot_${wfo}_${wvar}_${vhr}_${fhr}_${stats}_${ptype}_${period}.sh + case ${stats} in + 'stats1') + echo "export METRIC='me, rmse' " >> plot_${wfo}_${wvar}_${vhr}_${fhr}_${stats}_${ptype}_${period}.sh + ;; + 'stats2') + echo "export METRIC=pcor " >> plot_${wfo}_${wvar}_${vhr}_${fhr}_${stats}_${ptype}_${period}.sh + ;; + 'stats3') + echo "export METRIC='fbar, obar' " >> plot_${wfo}_${wvar}_${vhr}_${fhr}_${stats}_${ptype}_${period}.sh + ;; + 'stats4') + echo "export METRIC=esd " >> plot_${wfo}_${wvar}_${vhr}_${fhr}_${stats}_${ptype}_${period}.sh + ;; + 'stats5') + echo "export METRIC=si " >> plot_${wfo}_${wvar}_${vhr}_${fhr}_${stats}_${ptype}_${period}.sh + ;; + 'stats6') + echo "export METRIC=p95 " >> plot_${wfo}_${wvar}_${vhr}_${fhr}_${stats}_${ptype}_${period}.sh + ;; + esac + echo "export FHR=${fhr}" >> plot_${wfo}_${wvar}_${vhr}_${fhr}_${stats}_${ptype}_${period}.sh + echo "export WVAR=${wvar}" >> plot_${wfo}_${wvar}_${vhr}_${fhr}_${stats}_${ptype}_${period}.sh + case ${wvar} in + 'WIND') + echo "export OBS_LEVEL=Z10" >> plot_${wfo}_${wvar}_${vhr}_${fhr}_${stats}_${ptype}_${period}.sh + ;; + *) + echo "export OBS_LEVEL=L0" >> plot_${wfo}_${wvar}_${vhr}_${fhr}_${stats}_${ptype}_${period}.sh + ;; + esac + echo "export PTYPE=${ptype}" >> plot_${wfo}_${wvar}_${vhr}_${fhr}_${stats}_${ptype}_${period}.sh + echo "${GRID2OBS_CONF}/py_plotting_wave.config" >> plot_${wfo}_${wvar}_${vhr}_${fhr}_${stats}_${ptype}_${period}.sh + + chmod +x plot_${wfo}_${wvar}_${vhr}_${fhr}_${stats}_${ptype}_${period}.sh - echo "${DATA}/plot_${wvar}_${vhr}_${fhr}_${stats}_${ptype}_${period}.sh" >> plot_all_${MODELNAME}_${RUN}_g2o_plots.sh + echo "${DATA}/plot_${wfo}_${wvar}_${vhr}_${fhr}_${stats}_${ptype}_${period}.sh" >> plot_all_${MODELNAME}_${RUN}_g2o_${wfo}_plots.sh - done # fcst hrs - done # end of stats - done # end of wave vars - done # end of inithours + done # fcst hrs + done # end of stats + done # end of wave vars + done # end of inithours done # end of periods - diff --git a/ush/nwps/lead_average.py b/ush/nwps/lead_average.py index eeafebbbc..b3280caa3 100755 --- a/ush/nwps/lead_average.py +++ b/ush/nwps/lead_average.py @@ -48,7 +48,7 @@ model_colors = ModelSpecs() reference = Reference() - +WFO = os.environ ['WFO'] # =================== FUNCTIONS ========================= @@ -341,7 +341,11 @@ def plot_lead_average(df: pd.DataFrame, logger: logging.Logger, # Reindex pivot table with full list of lead hours, introducing NaNs x_vals_pre = pivot_metric1.index.tolist() lead_time_incr = np.diff(x_vals_pre) - min_incr = np.min(lead_time_incr) + if lead_time_incr.size > 0: + min_incr = np.min(lead_time_incr) + else: + min_incr = 1 + incrs = [1,6,12,24] incr_idx = np.digitize(min_incr, incrs) if incr_idx < 1: @@ -751,16 +755,21 @@ def plot_lead_average(df: pd.DataFrame, logger: logging.Logger, ]).flatten() round_to_nearest_categories = y_range_categories/20. y_range = y_max-y_min - round_to_nearest = round_to_nearest_categories[ + if str(y_range) == '-inf': + ylim_min = 0 + ylim_max = 1 + yticks = np.arange(ylim_min, ylim_max, 0.05) + else: + round_to_nearest = round_to_nearest_categories[ np.digitize(y_range, y_range_categories[:-1]) - ] - ylim_min = np.floor(y_min/round_to_nearest)*round_to_nearest - ylim_max = np.ceil(y_max/round_to_nearest)*round_to_nearest - if len(str(ylim_min)) > 5 and np.abs(ylim_min) < 1.: - ylim_min = float( - np.format_float_scientific(ylim_min, unique=False, precision=3) - ) - yticks = np.arange(ylim_min, ylim_max+round_to_nearest, round_to_nearest) + ] + ylim_min = np.floor(y_min/round_to_nearest)*round_to_nearest + ylim_max = np.ceil(y_max/round_to_nearest)*round_to_nearest + if len(str(ylim_min)) > 5 and np.abs(ylim_min) < 1.: + ylim_min = float( + np.format_float_scientific(ylim_min, unique=False, precision=3) + ) + yticks = np.arange(ylim_min, ylim_max+round_to_nearest, round_to_nearest) var_long_name_key = df['FCST_VAR'].tolist()[0] if str(var_long_name_key).upper() == 'HGT': if str(df['OBS_VAR'].tolist()[0]).upper() == 'CEILING': @@ -1026,7 +1035,8 @@ def plot_lead_average(df: pd.DataFrame, logger: logging.Logger, + f'{str(var_savename).lower()}_{str(level_savename).lower()}_{str(obtype).lower()}.' + f'{str(time_period_savename).lower()}.' + f'fhrmean_{str(date_type).lower()}{str(date_hours_savename).lower()}_f{xticks[-1]}.' - + f'{str(domain_string).lower()}') + + f'{str(domain_string).lower()}_' + + f'{str(WFO).lower()}') if metric2_name is not None: save_name = (f'evs.' + f'{str(models_savename).lower()}.' @@ -1034,7 +1044,8 @@ def plot_lead_average(df: pd.DataFrame, logger: logging.Logger, + f'{str(var_savename).lower()}_{str(level_savename).lower()}_{str(obtype).lower()}.' + f'{str(time_period_savename).lower()}.' + f'fhrmean_{str(date_type).lower()}{str(date_hours_savename).lower()}_f{xticks[-1]}.' - + f'{str(domain_string).lower()}') + + f'{str(domain_string).lower()}_' + + f'{str(WFO).lower()}') if thresh and '' not in thresh: save_name = (f'evs.' + f'{str(models_savename).lower()}.' @@ -1042,7 +1053,8 @@ def plot_lead_average(df: pd.DataFrame, logger: logging.Logger, + f'{str(var_savename).lower()}_{str(level_savename).lower()}_{str(obtype).lower()}.' + f'{str(time_period_savename).lower()}.' + f'fhrmean_{str(date_type).lower()}{str(date_hours_savename).lower()}_f{xticks[-1]}.' - + f'{str(domain).lower()}') + + f'{str(domain).lower()}_' + + f'{str(WFO).lower()}') if save_header: save_name = f'{save_header}_'+save_name save_subdir = os.path.join( diff --git a/ush/nwps/time_series.py b/ush/nwps/time_series.py index e504f8461..6cbf5b942 100755 --- a/ush/nwps/time_series.py +++ b/ush/nwps/time_series.py @@ -48,7 +48,7 @@ model_colors = ModelSpecs() reference = Reference() - +WFO = os.environ ['WFO'] # =================== FUNCTIONS ========================= @@ -760,16 +760,21 @@ def plot_time_series(df: pd.DataFrame, logger: logging.Logger, ]).flatten() round_to_nearest_categories = y_range_categories/20. y_range = y_max-y_min - round_to_nearest = round_to_nearest_categories[ + if str(y_range) == '-inf': + ylim_min = 0 + ylim_max = 1 + yticks = np.arange(ylim_min, ylim_max, 0.05) + else: + round_to_nearest = round_to_nearest_categories[ np.digitize(y_range, y_range_categories[:-1]) - ] - ylim_min = np.floor(y_min/round_to_nearest)*round_to_nearest - ylim_max = np.ceil(y_max/round_to_nearest)*round_to_nearest - if len(str(ylim_min)) > 5 and np.abs(ylim_min) < 1.: - ylim_min = float( - np.format_float_scientific(ylim_min, unique=False, precision=3) - ) - yticks = np.arange(ylim_min, ylim_max+round_to_nearest, round_to_nearest) + ] + ylim_min = np.floor(y_min/round_to_nearest)*round_to_nearest + ylim_max = np.ceil(y_max/round_to_nearest)*round_to_nearest + if len(str(ylim_min)) > 5 and np.abs(ylim_min) < 1.: + ylim_min = float( + np.format_float_scientific(ylim_min, unique=False, precision=3) + ) + yticks = np.arange(ylim_min, ylim_max+round_to_nearest, round_to_nearest) var_long_name_key = df['FCST_VAR'].tolist()[0] if str(var_long_name_key).upper() == 'HGT': if str(df['OBS_VAR'].tolist()[0]).upper() == 'CEILING': @@ -1036,7 +1041,8 @@ def plot_time_series(df: pd.DataFrame, logger: logging.Logger, + f'{str(time_period_savename).lower()}.' + f'timeseries_{str(date_type).lower()}{str(date_hours_savename).lower()}_' + f'{str(frange_save_string).lower()}.' - + f'{str(domain_string).lower()}') + + f'{str(domain_string).lower()}_' + + f'{str(WFO).lower()}') if metric2_name is not None: save_name = (f'evs.' + f'{str(models_savename).lower()}.' @@ -1045,7 +1051,8 @@ def plot_time_series(df: pd.DataFrame, logger: logging.Logger, + f'{str(time_period_savename).lower()}.' + f'timeseries_{str(date_type).lower()}{str(date_hours_savename).lower()}_' + f'{str(frange_save_string).lower()}.' - + f'{str(domain_string).lower()}') + + f'{str(domain_string).lower()}_' + + f'{str(WFO).lower()}') if thresh and '' not in thresh: save_name = (f'evs.' + f'{str(models_savename).lower()}.' @@ -1054,7 +1061,8 @@ def plot_time_series(df: pd.DataFrame, logger: logging.Logger, + f'{str(time_period_savename).lower()}.' + f'timeseries_{str(date_type).lower()}{str(date_hours_savename).lower()}_' + f'{str(frange_save_string).lower()}.' - + f'{str(domain_string).lower()}') + + f'{str(domain_string).lower()}_' + + f'{str(WFO).lower()}') if save_header: save_name = f'{save_header}_'+save_name save_subdir = os.path.join( From 5060d4a02a63bda45cc7f6646bd815e14904afac Mon Sep 17 00:00:00 2001 From: ShannonShields-NOAA <66634320+ShannonShields-NOAA@users.noreply.github.com> Date: Mon, 25 Nov 2024 14:30:49 -0500 Subject: [PATCH 6/6] Feature/subseasonal mpmd plots fix (#611) * Start MPMD changes for plot scripts * Change final output dir to tar_files * Start editing ush plots scripts for MPMD output dir fix * Edit plots_job_scripts.py for MPMD output dir changes * Continue making MPMD output dir changes for plot scripts * Finish making changes for MPMD plots bugzilla * Make logging_dir in subseasonal_plots.py * Make fixes in subseasonal_plots.py * Remove old/commented out code --------- Co-authored-by: shannon shields Co-authored-by: shannon shields Co-authored-by: shannon shields --- .../exevs_subseasonal_grid2grid_plots.sh | 33 ++++--- .../exevs_subseasonal_grid2obs_plots.sh | 33 ++++--- .../create_METplus_subseasonal_output_dirs.py | 8 +- .../subseasonal_copy_job_dir_output.py | 22 ++--- ush/subseasonal/subseasonal_plots.py | 98 +++++++++++-------- ...onal_plots_grid2grid_create_job_scripts.py | 54 ++++++++-- ...sonal_plots_grid2obs_create_job_scripts.py | 54 ++++++++-- .../subseasonal_plots_lead_average.py | 2 +- .../subseasonal_plots_performance_diagram.py | 2 +- .../subseasonal_plots_time_series.py | 2 +- 10 files changed, 205 insertions(+), 103 deletions(-) diff --git a/scripts/plots/subseasonal/exevs_subseasonal_grid2grid_plots.sh b/scripts/plots/subseasonal/exevs_subseasonal_grid2grid_plots.sh index 0341d3938..e9cd656e5 100755 --- a/scripts/plots/subseasonal/exevs_subseasonal_grid2grid_plots.sh +++ b/scripts/plots/subseasonal/exevs_subseasonal_grid2grid_plots.sh @@ -75,27 +75,32 @@ for group in condense_stats filter_stats make_plots tar_images; do nc=$((nc+1)) done fi + python $USHevs/subseasonal/subseasonal_copy_job_dir_output.py + export err=$?; err_chk + # Cat the plotting log files + if [ $JOB_GROUP = make_plots ]; then + log_dir=$DATA/${VERIF_CASE}_${STEP}/plot_output/job_work_dir/${JOB_GROUP}/job*/*/*/*/*/logs + else + log_dir=$DATA/${VERIF_CASE}_${STEP}/plot_output/job_work_dir/${JOB_GROUP}/job*/*/*/*/logs + fi + log_file_count=$(find $log_dir -type f |wc -l) + if [[ $log_file_count -ne 0 ]]; then + for log_file in $log_dir/*; do + echo "Start: $log_file" + cat $log_file + echo "End: $log_file" + done + fi done -# Cat the plotting log files -log_dir=$DATA/${VERIF_CASE}_${STEP}/plot_output/${RUN}.${end_date}/logs -log_file_count=$(find $log_dir -type f |wc -l) -if [[ $log_file_count -ne 0 ]]; then - for log_file in $log_dir/*; do - echo "Start: $log_file" - cat $log_file - echo "End: $log_file" - done -fi - # Copy files to desired location if [ $SENDCOM = YES ]; then # Make and copy tar file - cd $DATA/$VERIF_CASE_STEP/plot_output/${RUN}.${end_date}/images - for VERIF_TYPE_SUBDIR_PATH in $DATA/$VERIF_CASE_STEP/plot_output/$RUN.${end_date}/images/*; do + cd $DATA/$VERIF_CASE_STEP/plot_output/${RUN}.${end_date}/tar_files + for VERIF_TYPE_SUBDIR_PATH in $DATA/$VERIF_CASE_STEP/plot_output/$RUN.${end_date}/tar_files/*; do VERIF_TYPE_SUBDIR=$(echo ${VERIF_TYPE_SUBDIR_PATH##*/}) cd $VERIF_TYPE_SUBDIR - large_tar_file=${DATA}/${VERIF_CASE_STEP}/plot_output/${RUN}.${end_date}/images/evs.plots.${COMPONENT}.${RUN}.${VERIF_CASE}_${VERIF_TYPE_SUBDIR}.last${NDAYS}days.v${end_date}.tar + large_tar_file=${DATA}/${VERIF_CASE_STEP}/plot_output/${RUN}.${end_date}/tar_files/evs.plots.${COMPONENT}.${RUN}.${VERIF_CASE}_${VERIF_TYPE_SUBDIR}.last${NDAYS}days.v${end_date}.tar tar -cvf $large_tar_file *.tar if [ -s $large_tar_file ]; then cp -v $large_tar_file $COMOUT/. diff --git a/scripts/plots/subseasonal/exevs_subseasonal_grid2obs_plots.sh b/scripts/plots/subseasonal/exevs_subseasonal_grid2obs_plots.sh index ff708a8f0..e2175fd31 100755 --- a/scripts/plots/subseasonal/exevs_subseasonal_grid2obs_plots.sh +++ b/scripts/plots/subseasonal/exevs_subseasonal_grid2obs_plots.sh @@ -75,27 +75,32 @@ for group in condense_stats filter_stats make_plots tar_images; do nc=$((nc+1)) done fi + python $USHevs/subseasonal/subseasonal_copy_job_dir_output.py + export err=$?; err_chk + # Cat the plotting log files + if [ $JOB_GROUP = make_plots ]; then + log_dir=$DATA/${VERIF_CASE}_${STEP}/plot_output/job_work_dir/${JOB_GROUP}/job*/*/*/*/*/logs + else + log_dir=$DATA/${VERIF_CASE}_${STEP}/plot_output/job_work_dir/${JOB_GROUP}/job*/*/*/*/logs + fi + log_file_count=$(find $log_dir -type f |wc -l) + if [[ $log_file_count -ne 0 ]]; then + for log_file in $log_dir/*; do + echo "Start: $log_file" + cat $log_file + echo "End: $log_file" + done + fi done -# Cat the plotting log files -log_dir=$DATA/${VERIF_CASE}_${STEP}/plot_output/${RUN}.${end_date}/logs -log_file_count=$(find $log_dir -type f |wc -l) -if [[ $log_file_count -ne 0 ]]; then - for log_file in $log_dir/*; do - echo "Start: $log_file" - cat $log_file - echo "End: $log_file" - done -fi - # Copy files to desired location if [ $SENDCOM = YES ]; then # Make and copy tar file - cd $DATA/$VERIF_CASE_STEP/plot_output/${RUN}.${end_date}/images - for VERIF_TYPE_SUBDIR_PATH in $DATA/$VERIF_CASE_STEP/plot_output/$RUN.${end_date}/images/*; do + cd $DATA/$VERIF_CASE_STEP/plot_output/${RUN}.${end_date}/tar_files + for VERIF_TYPE_SUBDIR_PATH in $DATA/$VERIF_CASE_STEP/plot_output/$RUN.${end_date}/tar_files/*; do VERIF_TYPE_SUBDIR=$(echo ${VERIF_TYPE_SUBDIR_PATH##*/}) cd $VERIF_TYPE_SUBDIR - large_tar_file=${DATA}/${VERIF_CASE_STEP}/plot_output/${RUN}.${end_date}/images/evs.plots.${COMPONENT}.${RUN}.${VERIF_CASE}_${VERIF_TYPE_SUBDIR}.last${NDAYS}days.v${end_date}.tar + large_tar_file=${DATA}/${VERIF_CASE_STEP}/plot_output/${RUN}.${end_date}/tar_files/evs.plots.${COMPONENT}.${RUN}.${VERIF_CASE}_${VERIF_TYPE_SUBDIR}.last${NDAYS}days.v${end_date}.tar tar -cvf $large_tar_file *.tar if [ -s $large_tar_file ]; then cp -v $large_tar_file $COMOUT/. diff --git a/ush/subseasonal/create_METplus_subseasonal_output_dirs.py b/ush/subseasonal/create_METplus_subseasonal_output_dirs.py index 30dfe2935..6a3fa2f2c 100644 --- a/ush/subseasonal/create_METplus_subseasonal_output_dirs.py +++ b/ush/subseasonal/create_METplus_subseasonal_output_dirs.py @@ -120,19 +120,17 @@ 'plot_output') working_dir_list.append(working_output_base_dir) working_dir_list.append( - os.path.join(working_output_base_dir, - RUN+'.'+end_date_dt.strftime('%Y%m%d')) + os.path.join(working_output_base_dir, 'job_work_dir') ) working_dir_list.append( os.path.join(working_output_base_dir, - RUN+'.'+end_date_dt.strftime('%Y%m%d'), - 'logs') + RUN+'.'+end_date_dt.strftime('%Y%m%d')) ) for VCS_type in VCS_type_list: working_dir_list.append( os.path.join(working_output_base_dir, RUN+'.'+end_date_dt.strftime('%Y%m%d'), - 'images', VCS_type) + 'tar_files', VCS_type) ) working_dir_list.append( os.path.join(working_output_base_dir, diff --git a/ush/subseasonal/subseasonal_copy_job_dir_output.py b/ush/subseasonal/subseasonal_copy_job_dir_output.py index 299d55357..7d72c04be 100644 --- a/ush/subseasonal/subseasonal_copy_job_dir_output.py +++ b/ush/subseasonal/subseasonal_copy_job_dir_output.py @@ -19,8 +19,6 @@ STEP = os.environ['STEP'] COMPONENT = os.environ['COMPONENT'] JOB_GROUP = os.environ['JOB_GROUP'] -if STEP == 'plots': - NDAYS = os.environ['NDAYS'] # Copy files to desired location if STEP == 'stats': @@ -44,28 +42,24 @@ if JOB_GROUP == 'condense_stats': job_wildcard_dir = os.path.join( job_work_JOB_GROUP_dir, 'job*', f"{RUN}.*", '*', - f"last{NDAYS}days", '*', '*', '*', 'condensed_stats*' + '*', 'model1_*.stat' ) elif JOB_GROUP == 'filter_stats': job_wildcard_dir = os.path.join( job_work_JOB_GROUP_dir, 'job*', f"{RUN}.*", '*', - f"last{NDAYS}days", '*', '*', '*', 'fcst*.stat' + '*', 'fcst*.stat' ) elif JOB_GROUP == 'make_plots': job_wildcard_dir = os.path.join( job_work_JOB_GROUP_dir, 'job*', f"{RUN}.*", '*', - f"last{NDAYS}days", '*', '*', '*', '*', '*.png' + '*', '*', '*.png' + ) + elif JOB_GROUP == 'tar_images': + job_wildcard_dir = os.path.join( + job_work_JOB_GROUP_dir, 'job*', f"{RUN}.*", '*', + '*', '*.tar' ) output_file_JOB_list = glob.glob(job_wildcard_dir) -if STEP == 'plots' and JOB_GROUP == 'make_plots': - job_wildcard_dir2 = os.path.join( - job_work_JOB_GROUP_dir, 'job*', f"{RUN}.*", '*', - f"last{NDAYS}days", '*', '*', '*', '*', '*.gif' - ) - output_file_JOB_list = ( - output_file_JOB_list - + glob.glob(job_wildcard_dir2) - ) for output_file_JOB in sorted(output_file_JOB_list, key=len): output_file_end_path = output_file_JOB.partition( job_work_JOB_GROUP_dir+'/' diff --git a/ush/subseasonal/subseasonal_plots.py b/ush/subseasonal/subseasonal_plots.py index bf642f7a1..84235f1db 100644 --- a/ush/subseasonal/subseasonal_plots.py +++ b/ush/subseasonal/subseasonal_plots.py @@ -64,9 +64,14 @@ fhr_end = os.environ['fhr_end'] fhr_inc = os.environ['fhr_inc'] job_id = os.environ['job_id'] +job_DATA_dir = os.environ['job_DATA_dir'] if JOB_GROUP == 'make_plots': + job_DATA_images_dir = os.environ['job_DATA_images_dir'] + job_work_images_dir = os.environ['job_work_images_dir'] stat = os.environ['stat'] plot = os.environ['plot'] +else: + job_work_dir = os.environ['job_work_dir'] # Set variables VERIF_CASE_STEP = VERIF_CASE+'_'+STEP @@ -78,14 +83,17 @@ logo_dir = os.path.join(FIXevs, 'logos') VERIF_CASE_STEP_dir = os.path.join(DATA, VERIF_CASE_STEP) stat_base_dir = os.path.join(VERIF_CASE_STEP_dir, 'data') -plot_output_dir = os.path.join(VERIF_CASE_STEP_dir, 'plot_output') -logging_dir = os.path.join(plot_output_dir, RUN+'.'+end_date, 'logs') -VERIF_TYPE_image_dir = os.path.join(plot_output_dir, RUN+'.'+end_date, - 'images', VERIF_TYPE) -job_output_dir = os.path.join(plot_output_dir, RUN+'.'+end_date, - VERIF_TYPE, job_name.replace('/','_')) -if not os.path.exists(job_output_dir): - os.makedirs(job_output_dir) +if JOB_GROUP == 'tar_images': + VERIF_TYPE_tar_dir = os.path.join(job_work_dir, RUN+'.'+end_date, + 'tar_files', VERIF_TYPE) +if JOB_GROUP == 'make_plots': + logging_dir = os.path.join(job_work_images_dir, 'logs') +elif JOB_GROUP == 'tar_images': + logging_dir = os.path.join(VERIF_TYPE_tar_dir, 'logs') +else: + logging_dir = os.path.join(job_work_dir, 'logs') +if not os.path.exists(logging_dir): + os.makedirs(logging_dir) # Set up logging job_logging_file = os.path.join(logging_dir, 'evs_'+COMPONENT+'_'+RUN+'_' @@ -172,13 +180,14 @@ for model_idx in range(len(model_list)): model = model_list[model_idx] obs_name = obs_list[model_idx] - condensed_model_stat_file = os.path.join(job_output_dir, 'model' + condensed_model_stat_file = os.path.join(job_work_dir, 'model' +str(model_idx+1)+'_'+model +'.stat') sub_util.condense_model_stat_files(logger, stat_base_dir, condensed_model_stat_file, model, obs_name, grid, vx_mask, - fcst_var_name, obs_var_name, line_type) + fcst_var_name, obs_var_name, + line_type) elif JOB_GROUP == 'filter_stats': logger.info("Filtering model .stat files") model_info_dict = original_model_info_dict.copy() @@ -241,7 +250,7 @@ else: plot_dates = valid_dates all_model_df = sub_util.build_df( - logger, job_output_dir, job_output_dir, + logger, job_DATA_dir, job_work_dir, model_info_dict, met_info_dict, plot_info_dict['fcst_var_name'], plot_info_dict['fcst_var_level'], @@ -292,7 +301,7 @@ int(date_info_dict['forecast_hour']) ) image_name = plot_specs.get_savefig_name( - os.path.join(job_output_dir, 'images'), + job_work_images_dir, plot_info_dict, date_info_dict ) if init_hr in init_hrs: @@ -303,8 +312,9 @@ else: make_ts = False if make_ts: - plot_ts = subp_ts.TimeSeries(logger, job_output_dir, - job_output_dir, model_info_dict, + plot_ts = subp_ts.TimeSeries(logger, job_DATA_dir, + job_work_images_dir, + model_info_dict, date_info_dict, plot_info_dict, met_info_dict, logo_dir) plot_ts.make_time_series() @@ -325,7 +335,7 @@ plot_info_dict['obs_var_thresh'] = la_info[1][1][2] plot_info_dict['interp_points'] = str(la_info[2]) image_name = plot_specs.get_savefig_name( - os.path.join(job_output_dir, 'images'), + job_work_images_dir, plot_info_dict, date_info_dict ) if not os.path.exists(image_name): @@ -342,8 +352,9 @@ else: make_la = False if make_la: - plot_la = subp_la.LeadAverage(logger, job_output_dir, - job_output_dir, model_info_dict, + plot_la = subp_la.LeadAverage(logger, job_DATA_dir, + job_work_images_dir, + model_info_dict, date_info_dict, plot_info_dict, met_info_dict, logo_dir) plot_la.make_lead_average() @@ -363,7 +374,7 @@ plot_info_dict['obs_var_thresh'] = vha_info[0][1][2] plot_info_dict['interp_points'] = str(vha_info[1]) image_name = plot_specs.get_savefig_name( - os.path.join(job_output_dir, 'images'), + job_work_images_dir, plot_info_dict, date_info_dict ) if not os.path.exists(image_name): @@ -382,8 +393,8 @@ else: make_vha = False if make_vha: - plot_vha = subp_vha.ValidHourAverage(logger, job_output_dir, - job_output_dir, + plot_vha = subp_vha.ValidHourAverage(logger, job_DATA_dir, + job_work_images_dir, model_info_dict, date_info_dict, plot_info_dict, @@ -411,7 +422,7 @@ plot_info_dict['fcst_var_level'] = fcst_var_level_list[l] plot_info_dict['obs_var_level'] = obs_var_level_list[l] image_name = plot_specs.get_savefig_name( - os.path.join(job_output_dir, 'images'), + job_work_images_dir, plot_info_dict, date_info_dict ) if init_hr in init_hrs: @@ -431,8 +442,8 @@ else: make_ta = False if make_ta: - plot_ta = subp_ta.ThresholdAverage(logger, job_output_dir, - job_output_dir, + plot_ta = subp_ta.ThresholdAverage(logger, job_DATA_dir, + job_work_images_dir, model_info_dict, date_info_dict, plot_info_dict, @@ -456,7 +467,7 @@ plot_info_dict['obs_var_thresh'] = lbd_info[1][1][2] plot_info_dict['interp_points'] = str(lbd_info[2]) image_name = plot_specs.get_savefig_name( - os.path.join(job_output_dir, 'images'), + job_work_images_dir, plot_info_dict, date_info_dict ) if not os.path.exists(image_name): @@ -473,8 +484,9 @@ else: make_lbd = False if make_lbd: - plot_lbd = subp_lbd.LeadByDate(logger, job_output_dir, - job_output_dir, model_info_dict, + plot_lbd = subp_lbd.LeadByDate(logger, job_DATA_dir, + job_work_images_dir, + model_info_dict, date_info_dict, plot_info_dict, met_info_dict, logo_dir) plot_lbd.make_lead_by_date() @@ -502,7 +514,7 @@ plot_info_dict['fcst_var_thresh'] = fcst_var_thresh_list[t] plot_info_dict['obs_var_thresh'] = obs_var_thresh_list[t] image_name = plot_specs.get_savefig_name( - os.path.join(job_output_dir, 'images'), + job_work_images_dir, plot_info_dict, date_info_dict ) if init_hr in init_hrs: @@ -518,8 +530,8 @@ del plot_info_dict['fcst_var_level'] del plot_info_dict['obs_var_level'] if make_sbl: - plot_sbl = subp_sbl.StatByLevel(logger, job_output_dir, - job_output_dir, + plot_sbl = subp_sbl.StatByLevel(logger, job_DATA_dir, + job_work_images_dir, model_info_dict, date_info_dict, plot_info_dict, @@ -552,7 +564,7 @@ plot_info_dict['fcst_var_thresh'] = fcst_var_thresh_list[t] plot_info_dict['obs_var_thresh'] = obs_var_thresh_list[t] image_name = plot_specs.get_savefig_name( - os.path.join(job_output_dir, 'images'), + job_work_images_dir, plot_info_dict, date_info_dict ) if not os.path.exists(image_name): @@ -570,8 +582,8 @@ del plot_info_dict['fcst_var_level'] del plot_info_dict['obs_var_level'] if make_lbl: - plot_lbl = subp_lbl.LeadByLevel(logger, job_output_dir, - job_output_dir, + plot_lbl = subp_lbl.LeadByLevel(logger, job_DATA_dir, + job_work_images_dir, model_info_dict, date_info_dict, plot_info_dict, @@ -599,7 +611,7 @@ plot_info_dict['fcst_var_level'] = fcst_var_level_list[l] plot_info_dict['obs_var_level'] = obs_var_level_list[l] image_name = plot_specs.get_savefig_name( - os.path.join(job_output_dir, 'images'), + job_work_images_dir, plot_info_dict, date_info_dict ) if init_hr in init_hrs: @@ -610,8 +622,8 @@ else: make_pd = False if make_pd: - plot_pd = subp_pd.PerformanceDiagram(logger, job_output_dir, - job_output_dir, + plot_pd = subp_pd.PerformanceDiagram(logger, job_DATA_dir, + job_work_images_dir, model_info_dict, date_info_dict, plot_info_dict, @@ -624,21 +636,25 @@ # Create tar file of plots and move to main image directory elif JOB_GROUP == 'tar_images': logger.info("Tar'ing up images") - job_output_image_dir = os.path.join(job_output_dir, 'images') + job_input_image_dir = os.path.join( + DATA, VERIF_CASE+'_'+STEP, 'plot_output', + RUN+'.'+end_date, VERIF_TYPE, + job_name.replace('/','_'), 'images' + ) cwd = os.getcwd() - if len(glob.glob(job_output_image_dir+'/*')) != 0: - os.chdir(job_output_image_dir) - tar_file = os.path.join(VERIF_TYPE_image_dir, + if len(glob.glob(job_input_image_dir+'/*')) != 0: + os.chdir(job_input_image_dir) + tar_file = os.path.join(VERIF_TYPE_tar_dir, job_name.replace('/','_')+'.tar') if os.path.exists(tar_file): os.remove(tar_file) - logger.debug(f"Making tar file {tar_file} from {job_output_image_dir}") + logger.debug(f"Making tar file {tar_file} from {job_input_image_dir}") sub_util.run_shell_command( ['tar', '-cvf', tar_file, '*'] ) os.chdir(cwd) else: - logger.warning(f"No images generated in {job_output_image_dir}") + logger.warning(f"No images generated in {job_input_image_dir}") print("END: "+os.path.basename(__file__)) diff --git a/ush/subseasonal/subseasonal_plots_grid2grid_create_job_scripts.py b/ush/subseasonal/subseasonal_plots_grid2grid_create_job_scripts.py index b5ace06a6..bf05b5dd2 100644 --- a/ush/subseasonal/subseasonal_plots_grid2grid_create_job_scripts.py +++ b/ush/subseasonal/subseasonal_plots_grid2grid_create_job_scripts.py @@ -554,13 +554,14 @@ +verif_type_job+'/' +job_env_dict['vx_mask'] ) - job_output_dir = os.path.join( + job_DATA_dir = os.path.join( DATA, VERIF_CASE+'_'+STEP, 'plot_output', RUN+'.'+end_date, verif_type, job_env_dict['job_name'].replace('/','_') ) - if not os.path.exists(job_output_dir): - os.makedirs(job_output_dir) + job_env_dict['job_DATA_dir'] = job_DATA_dir + if not os.path.exists(job_DATA_dir): + os.makedirs(job_DATA_dir) if JOB_GROUP == 'condense_stats': for JOB_GROUP_loop in list( itertools.product(model_list, [fcst_var_levels], @@ -598,10 +599,18 @@ # Set any environment variables for special cases # Write environment variables job_env_dict['job_id'] = 'job'+str(njobs) + job_env_dict['job_work_dir'] = os.path.join( + DATA, VERIF_CASE+'_'+STEP, 'plot_output', + 'job_work_dir', JOB_GROUP, + f"{job_env_dict['job_id']}", RUN+'.'+end_date, + verif_type, job_env_dict['job_name'].replace('/','_') + ) for name, value in job_env_dict.items(): job.write('export '+name+'='+value+'\n') job.write('\n') # Write job commands + if not os.path.exists(job_env_dict['job_work_dir']): + os.makedirs(job_env_dict['job_work_dir']) job.write( sub_util.python_command('subseasonal_plots.py',[]) +'\n' @@ -650,9 +659,17 @@ job.write('\n') # Write environment variables job_env_dict['job_id'] = 'job'+str(njobs) + job_env_dict['job_work_dir'] = os.path.join( + DATA, VERIF_CASE+'_'+STEP, 'plot_output', + 'job_work_dir', JOB_GROUP, + f"{job_env_dict['job_id']}", RUN+'.'+end_date, + verif_type, job_env_dict['job_name'].replace('/','_') + ) for name, value in job_env_dict.items(): job.write('export '+name+'='+value+'\n') job.write('\n') + if not os.path.exists(job_env_dict['job_work_dir']): + os.makedirs(job_env_dict['job_work_dir']) job.write( sub_util.python_command('subseasonal_plots.py',[]) +'\n' @@ -660,13 +677,14 @@ job.write('export err=$?; err_chk'+'\n') job.close() elif JOB_GROUP == 'make_plots': - job_output_images_dir = os.path.join( + job_DATA_images_dir = os.path.join( DATA, VERIF_CASE+'_'+STEP, 'plot_output', RUN+'.'+end_date, verif_type, job_env_dict['job_name'].replace('/','_'), 'images' ) - if not os.path.exists(job_output_images_dir): - os.makedirs(job_output_images_dir) + job_env_dict['job_DATA_images_dir'] = job_DATA_images_dir + if not os.path.exists(job_DATA_images_dir): + os.makedirs(job_DATA_images_dir) job_env_dict['model_list'] = "'"+f"{', '.join(model_list)}"+"'" job_env_dict['model_plot_name_list'] = ( "'"+f"{', '.join(model_plot_name_list)}"+"'" @@ -756,9 +774,19 @@ job.write('\n') # Write environment variables job_env_dict['job_id'] = 'job'+str(njobs) + job_env_dict['job_work_images_dir'] = os.path.join( + DATA, VERIF_CASE+'_'+STEP, 'plot_output', + 'job_work_dir', JOB_GROUP, + f"{job_env_dict['job_id']}", RUN+'.'+end_date, + verif_type, + job_env_dict['job_name'].replace('/','_'), + 'images' + ) for name, value in job_env_dict.items(): job.write('export '+name+'='+value+'\n') job.write('\n') + if not os.path.exists(job_env_dict['job_work_images_dir']): + os.makedirs(job_env_dict['job_work_images_dir']) job.write( sub_util.python_command(run_subseasonal_plot, [])+'\n' @@ -766,6 +794,13 @@ job.write('export err=$?; err_chk'+'\n') job.close() elif JOB_GROUP == 'tar_images': + job_DATA_dir = os.path.join( + DATA, VERIF_CASE+'_'+STEP, 'plot_output', + RUN+'.'+end_date, 'tar_files', verif_type + ) + job_env_dict['job_DATA_dir'] = job_DATA_dir + if not os.path.exists(job_env_dict['job_DATA_dir']): + os.makedirs(job_env_dict['job_DATA_dir']) job_env_dict['model_list'] = "'"+f"{', '.join(model_list)}"+"'" job_env_dict['model_plot_name_list'] = ( "'"+f"{', '.join(model_plot_name_list)}"+"'" @@ -797,9 +832,16 @@ job.write('\n') # Write environment variables job_env_dict['job_id'] = 'job'+str(njobs) + job_env_dict['job_work_dir'] = os.path.join( + DATA, VERIF_CASE+'_'+STEP, 'plot_output', + 'job_work_dir', JOB_GROUP, + f"{job_env_dict['job_id']}" + ) for name, value in job_env_dict.items(): job.write('export '+name+'='+value+'\n') job.write('\n') + if not os.path.exists(job_env_dict['job_work_dir']): + os.makedirs(job_env_dict['job_work_dir']) job.write( sub_util.python_command('subseasonal_plots.py',[]) +'\n' diff --git a/ush/subseasonal/subseasonal_plots_grid2obs_create_job_scripts.py b/ush/subseasonal/subseasonal_plots_grid2obs_create_job_scripts.py index aaf38c997..108b3d1ae 100644 --- a/ush/subseasonal/subseasonal_plots_grid2obs_create_job_scripts.py +++ b/ush/subseasonal/subseasonal_plots_grid2obs_create_job_scripts.py @@ -258,13 +258,14 @@ +verif_type_job+'/' +job_env_dict['vx_mask'] ) - job_output_dir = os.path.join( + job_DATA_dir = os.path.join( DATA, VERIF_CASE+'_'+STEP, 'plot_output', RUN+'.'+end_date, verif_type, job_env_dict['job_name'].replace('/','_') ) - if not os.path.exists(job_output_dir): - os.makedirs(job_output_dir) + job_env_dict['job_DATA_dir'] = job_DATA_dir + if not os.path.exists(job_DATA_dir): + os.makedirs(job_DATA_dir) if JOB_GROUP == 'condense_stats': for JOB_GROUP_loop in list( itertools.product(model_list, [fcst_var_levels], @@ -302,10 +303,18 @@ # Set any environment variables for special cases # Write environment variables job_env_dict['job_id'] = 'job'+str(njobs) + job_env_dict['job_work_dir'] = os.path.join( + DATA, VERIF_CASE+'_'+STEP, 'plot_output', + 'job_work_dir', JOB_GROUP, + f"{job_env_dict['job_id']}", RUN+'.'+end_date, + verif_type, job_env_dict['job_name'].replace('/','_') + ) for name, value in job_env_dict.items(): job.write('export '+name+'='+value+'\n') job.write('\n') # Write job commands + if not os.path.exists(job_env_dict['job_work_dir']): + os.makedirs(job_env_dict['job_work_dir']) job.write( sub_util.python_command('subseasonal_plots.py',[]) +'\n' @@ -354,9 +363,17 @@ job.write('\n') # Write environment variables job_env_dict['job_id'] = 'job'+str(njobs) + job_env_dict['job_work_dir'] = os.path.join( + DATA, VERIF_CASE+'_'+STEP, 'plot_output', + 'job_work_dir', JOB_GROUP, + f"{job_env_dict['job_id']}", RUN+'.'+end_date, + verif_type, job_env_dict['job_name'].replace('/','_') + ) for name, value in job_env_dict.items(): job.write('export '+name+'='+value+'\n') job.write('\n') + if not os.path.exists(job_env_dict['job_work_dir']): + os.makedirs(job_env_dict['job_work_dir']) job.write( sub_util.python_command('subseasonal_plots.py',[]) +'\n' @@ -364,13 +381,14 @@ job.write('export err=$?; err_chk'+'\n') job.close() elif JOB_GROUP == 'make_plots': - job_output_images_dir = os.path.join( + job_DATA_images_dir = os.path.join( DATA, VERIF_CASE+'_'+STEP, 'plot_output', RUN+'.'+end_date, verif_type, job_env_dict['job_name'].replace('/','_'), 'images' ) - if not os.path.exists(job_output_images_dir): - os.makedirs(job_output_images_dir) + job_env_dict['job_DATA_images_dir'] = job_DATA_images_dir + if not os.path.exists(job_DATA_images_dir): + os.makedirs(job_DATA_images_dir) job_env_dict['model_list'] = "'"+f"{', '.join(model_list)}"+"'" job_env_dict['model_plot_name_list'] = ( "'"+f"{', '.join(model_plot_name_list)}"+"'" @@ -460,9 +478,19 @@ job.write('\n') # Write environment variables job_env_dict['job_id'] = 'job'+str(njobs) + job_env_dict['job_work_images_dir'] = os.path.join( + DATA, VERIF_CASE+'_'+STEP, 'plot_output', + 'job_work_dir', JOB_GROUP, + f"{job_env_dict['job_id']}", RUN+'.'+end_date, + verif_type, + job_env_dict['job_name'].replace('/','_'), + 'images' + ) for name, value in job_env_dict.items(): job.write('export '+name+'='+value+'\n') job.write('\n') + if not os.path.exists(job_env_dict['job_work_images_dir']): + os.makedirs(job_env_dict['job_work_images_dir']) job.write( sub_util.python_command(run_subseasonal_plot, [])+'\n' @@ -470,6 +498,13 @@ job.write('export err=$?; err_chk'+'\n') job.close() elif JOB_GROUP == 'tar_images': + job_DATA_dir = os.path.join( + DATA, VERIF_CASE+'_'+STEP, 'plot_output', + RUN+'.'+end_date, 'tar_files', verif_type + ) + job_env_dict['job_DATA_dir'] = job_DATA_dir + if not os.path.exists(job_env_dict['job_DATA_dir']): + os.makedirs(job_env_dict['job_DATA_dir']) job_env_dict['model_list'] = "'"+f"{', '.join(model_list)}"+"'" job_env_dict['model_plot_name_list'] = ( "'"+f"{', '.join(model_plot_name_list)}"+"'" @@ -501,9 +536,16 @@ job.write('\n') # Write environment variables job_env_dict['job_id'] = 'job'+str(njobs) + job_env_dict['job_work_dir'] = os.path.join( + DATA, VERIF_CASE+'_'+STEP, 'plot_output', + 'job_work_dir', JOB_GROUP, + f"{job_env_dict['job_id']}" + ) for name, value in job_env_dict.items(): job.write('export '+name+'='+value+'\n') job.write('\n') + if not os.path.exists(job_env_dict['job_work_dir']): + os.makedirs(job_env_dict['job_work_dir']) job.write( sub_util.python_command('subseasonal_plots.py',[]) +'\n' diff --git a/ush/subseasonal/subseasonal_plots_lead_average.py b/ush/subseasonal/subseasonal_plots_lead_average.py index af4ad1dd1..f129fe5c4 100644 --- a/ush/subseasonal/subseasonal_plots_lead_average.py +++ b/ush/subseasonal/subseasonal_plots_lead_average.py @@ -74,7 +74,7 @@ def make_lead_average(self): +f"{self.plot_info_dict['stat']}") sys.exit(0) # Make job image directory - output_image_dir = os.path.join(self.output_dir, 'images') + output_image_dir = self.output_dir if not os.path.exists(output_image_dir): os.makedirs(output_image_dir) self.logger.info(f"Plots will be in: {output_image_dir}") diff --git a/ush/subseasonal/subseasonal_plots_performance_diagram.py b/ush/subseasonal/subseasonal_plots_performance_diagram.py index 5efd725f6..37b7a10d3 100644 --- a/ush/subseasonal/subseasonal_plots_performance_diagram.py +++ b/ush/subseasonal/subseasonal_plots_performance_diagram.py @@ -74,7 +74,7 @@ def make_performance_diagram(self): +f"{self.plot_info_dict['stat']}") sys.exit(0) # Make job image directory - output_image_dir = os.path.join(self.output_dir, 'images') + output_image_dir = self.output_dir if not os.path.exists(output_image_dir): os.makedirs(output_image_dir) self.logger.info(f"Plots will be in: {output_image_dir}") diff --git a/ush/subseasonal/subseasonal_plots_time_series.py b/ush/subseasonal/subseasonal_plots_time_series.py index fb25e6576..546e08a14 100644 --- a/ush/subseasonal/subseasonal_plots_time_series.py +++ b/ush/subseasonal/subseasonal_plots_time_series.py @@ -69,7 +69,7 @@ def make_time_series(self): self.logger.debug(f"Plot information dictionary: " +f"{self.plot_info_dict}") # Make job image directory - output_image_dir = os.path.join(self.output_dir, 'images') + output_image_dir = self.output_dir if not os.path.exists(output_image_dir): os.makedirs(output_image_dir) self.logger.info(f"Plots will be in: {output_image_dir}")