From edc15670c7836b0bc459c10f88cdf39e23ca010e Mon Sep 17 00:00:00 2001 From: shannon shields Date: Mon, 18 Nov 2024 21:13:53 +0000 Subject: [PATCH 1/9] Start MPMD changes for plot scripts --- .../exevs_subseasonal_grid2grid_plots.sh | 34 ++++++++++++++----- .../exevs_subseasonal_grid2obs_plots.sh | 27 +++++++++------ 2 files changed, 41 insertions(+), 20 deletions(-) diff --git a/scripts/plots/subseasonal/exevs_subseasonal_grid2grid_plots.sh b/scripts/plots/subseasonal/exevs_subseasonal_grid2grid_plots.sh index 0341d39381..51a7b9fb36 100755 --- a/scripts/plots/subseasonal/exevs_subseasonal_grid2grid_plots.sh +++ b/scripts/plots/subseasonal/exevs_subseasonal_grid2grid_plots.sh @@ -75,18 +75,34 @@ for group in condense_stats filter_stats make_plots tar_images; do nc=$((nc+1)) done fi + python $USHevs/subseasonal/subseasonal_copy_job_dir_output.py + export err=$?; err_chk + # Cat the plotting log files + if [ $JOB_GROUP = make_plots ]; then + log_dir=$DATA/${VERIF_CASE}_${STEP}/plot_output/job_work_dir/${JOB_GROUP}/job*/*/*/*/*/*/*/*/logs + else + log_dir=$DATA/${VERIF_CASE}_${STEP}/plot_output/job_work_dir/${JOB_GROUP}/job*/*/*/*/*/*/*/logs + fi + log_file_count=$(find $log_dir -type f |wc -l) + if [[ $log_file_count -ne 0 ]]; then + for log_file in $log_dir/*; do + echo "Start: $log_file" + cat $log_file + echo "End: $log_file" + done + fi done # Cat the plotting log files -log_dir=$DATA/${VERIF_CASE}_${STEP}/plot_output/${RUN}.${end_date}/logs -log_file_count=$(find $log_dir -type f |wc -l) -if [[ $log_file_count -ne 0 ]]; then - for log_file in $log_dir/*; do - echo "Start: $log_file" - cat $log_file - echo "End: $log_file" - done -fi +#log_dir=$DATA/${VERIF_CASE}_${STEP}/plot_output/${RUN}.${end_date}/logs +#log_file_count=$(find $log_dir -type f |wc -l) +#if [[ $log_file_count -ne 0 ]]; then + #for log_file in $log_dir/*; do + #echo "Start: $log_file" + #cat $log_file + #echo "End: $log_file" + #done +#fi # Copy files to desired location if [ $SENDCOM = YES ]; then diff --git a/scripts/plots/subseasonal/exevs_subseasonal_grid2obs_plots.sh b/scripts/plots/subseasonal/exevs_subseasonal_grid2obs_plots.sh index ff708a8f0e..b2418b5c6c 100755 --- a/scripts/plots/subseasonal/exevs_subseasonal_grid2obs_plots.sh +++ b/scripts/plots/subseasonal/exevs_subseasonal_grid2obs_plots.sh @@ -75,19 +75,24 @@ for group in condense_stats filter_stats make_plots tar_images; do nc=$((nc+1)) done fi + python $USHevs/subseasonal/subseasonal_copy_job_dir_output.py + export err=$?; err_chk + # Cat the plotting log files + if [ $JOB_GROUP = make_plots ]; then + log_dir=$DATA/${VERIF_CASE}_${STEP}/plot_output/job_work_dir/${JOB_GROUP}/job*/*/*/*/*/*/*/*/logs + else + log_dir=$DATA/${VERIF_CASE}_${STEP}/plot_output/job_work_dir/${JOB_GROUP}/job*/*/*/*/*/*/*/logs + fi + log_file_count=$(find $log_dir -type f |wc -l) + if [[ $log_file_count -ne 0 ]]; then + for log_file in $log_dir/*; do + echo "Start: $log_file" + cat $log_file + echo "End: $log_file" + done + fi done -# Cat the plotting log files -log_dir=$DATA/${VERIF_CASE}_${STEP}/plot_output/${RUN}.${end_date}/logs -log_file_count=$(find $log_dir -type f |wc -l) -if [[ $log_file_count -ne 0 ]]; then - for log_file in $log_dir/*; do - echo "Start: $log_file" - cat $log_file - echo "End: $log_file" - done -fi - # Copy files to desired location if [ $SENDCOM = YES ]; then # Make and copy tar file From 43f720d529902ebd25d62f2ec83d35f13f33be9f Mon Sep 17 00:00:00 2001 From: shannon shields Date: Tue, 19 Nov 2024 19:41:32 +0000 Subject: [PATCH 2/9] Change final output dir to tar_files --- .../subseasonal/exevs_subseasonal_grid2grid_plots.sh | 6 +++--- .../subseasonal/exevs_subseasonal_grid2obs_plots.sh | 6 +++--- .../create_METplus_subseasonal_output_dirs.py | 11 +++++++---- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/scripts/plots/subseasonal/exevs_subseasonal_grid2grid_plots.sh b/scripts/plots/subseasonal/exevs_subseasonal_grid2grid_plots.sh index 51a7b9fb36..461891e370 100755 --- a/scripts/plots/subseasonal/exevs_subseasonal_grid2grid_plots.sh +++ b/scripts/plots/subseasonal/exevs_subseasonal_grid2grid_plots.sh @@ -107,11 +107,11 @@ done # Copy files to desired location if [ $SENDCOM = YES ]; then # Make and copy tar file - cd $DATA/$VERIF_CASE_STEP/plot_output/${RUN}.${end_date}/images - for VERIF_TYPE_SUBDIR_PATH in $DATA/$VERIF_CASE_STEP/plot_output/$RUN.${end_date}/images/*; do + cd $DATA/$VERIF_CASE_STEP/plot_output/${RUN}.${end_date}/tar_files + for VERIF_TYPE_SUBDIR_PATH in $DATA/$VERIF_CASE_STEP/plot_output/$RUN.${end_date}/tar_files/*; do VERIF_TYPE_SUBDIR=$(echo ${VERIF_TYPE_SUBDIR_PATH##*/}) cd $VERIF_TYPE_SUBDIR - large_tar_file=${DATA}/${VERIF_CASE_STEP}/plot_output/${RUN}.${end_date}/images/evs.plots.${COMPONENT}.${RUN}.${VERIF_CASE}_${VERIF_TYPE_SUBDIR}.last${NDAYS}days.v${end_date}.tar + large_tar_file=${DATA}/${VERIF_CASE_STEP}/plot_output/${RUN}.${end_date}/tar_files/evs.plots.${COMPONENT}.${RUN}.${VERIF_CASE}_${VERIF_TYPE_SUBDIR}.last${NDAYS}days.v${end_date}.tar tar -cvf $large_tar_file *.tar if [ -s $large_tar_file ]; then cp -v $large_tar_file $COMOUT/. diff --git a/scripts/plots/subseasonal/exevs_subseasonal_grid2obs_plots.sh b/scripts/plots/subseasonal/exevs_subseasonal_grid2obs_plots.sh index b2418b5c6c..f608085227 100755 --- a/scripts/plots/subseasonal/exevs_subseasonal_grid2obs_plots.sh +++ b/scripts/plots/subseasonal/exevs_subseasonal_grid2obs_plots.sh @@ -96,11 +96,11 @@ done # Copy files to desired location if [ $SENDCOM = YES ]; then # Make and copy tar file - cd $DATA/$VERIF_CASE_STEP/plot_output/${RUN}.${end_date}/images - for VERIF_TYPE_SUBDIR_PATH in $DATA/$VERIF_CASE_STEP/plot_output/$RUN.${end_date}/images/*; do + cd $DATA/$VERIF_CASE_STEP/plot_output/${RUN}.${end_date}/tar_files + for VERIF_TYPE_SUBDIR_PATH in $DATA/$VERIF_CASE_STEP/plot_output/$RUN.${end_date}/tar_files/*; do VERIF_TYPE_SUBDIR=$(echo ${VERIF_TYPE_SUBDIR_PATH##*/}) cd $VERIF_TYPE_SUBDIR - large_tar_file=${DATA}/${VERIF_CASE_STEP}/plot_output/${RUN}.${end_date}/images/evs.plots.${COMPONENT}.${RUN}.${VERIF_CASE}_${VERIF_TYPE_SUBDIR}.last${NDAYS}days.v${end_date}.tar + large_tar_file=${DATA}/${VERIF_CASE_STEP}/plot_output/${RUN}.${end_date}/tar_files/evs.plots.${COMPONENT}.${RUN}.${VERIF_CASE}_${VERIF_TYPE_SUBDIR}.last${NDAYS}days.v${end_date}.tar tar -cvf $large_tar_file *.tar if [ -s $large_tar_file ]; then cp -v $large_tar_file $COMOUT/. diff --git a/ush/subseasonal/create_METplus_subseasonal_output_dirs.py b/ush/subseasonal/create_METplus_subseasonal_output_dirs.py index 30dfe2935b..a4105cb91a 100644 --- a/ush/subseasonal/create_METplus_subseasonal_output_dirs.py +++ b/ush/subseasonal/create_METplus_subseasonal_output_dirs.py @@ -120,14 +120,17 @@ 'plot_output') working_dir_list.append(working_output_base_dir) working_dir_list.append( - os.path.join(working_output_base_dir, - RUN+'.'+end_date_dt.strftime('%Y%m%d')) + os.path.join(working_output_base_dir, 'job_work_dir') ) working_dir_list.append( os.path.join(working_output_base_dir, - RUN+'.'+end_date_dt.strftime('%Y%m%d'), - 'logs') + RUN+'.'+end_date_dt.strftime('%Y%m%d')) ) + #working_dir_list.append( + #os.path.join(working_output_base_dir, + #RUN+'.'+end_date_dt.strftime('%Y%m%d'), + #'logs') + #) for VCS_type in VCS_type_list: working_dir_list.append( os.path.join(working_output_base_dir, From 52fd9df2aa87889d355b2be285ca8d7ae3e4b4d0 Mon Sep 17 00:00:00 2001 From: shannon shields Date: Tue, 19 Nov 2024 20:40:59 +0000 Subject: [PATCH 3/9] Start editing ush plots scripts for MPMD output dir fix --- .../create_METplus_subseasonal_output_dirs.py | 2 +- .../subseasonal_copy_job_dir_output.py | 22 +++++++------------ 2 files changed, 9 insertions(+), 15 deletions(-) diff --git a/ush/subseasonal/create_METplus_subseasonal_output_dirs.py b/ush/subseasonal/create_METplus_subseasonal_output_dirs.py index a4105cb91a..9ff085995d 100644 --- a/ush/subseasonal/create_METplus_subseasonal_output_dirs.py +++ b/ush/subseasonal/create_METplus_subseasonal_output_dirs.py @@ -135,7 +135,7 @@ working_dir_list.append( os.path.join(working_output_base_dir, RUN+'.'+end_date_dt.strftime('%Y%m%d'), - 'images', VCS_type) + 'tar_files', VCS_type) ) working_dir_list.append( os.path.join(working_output_base_dir, diff --git a/ush/subseasonal/subseasonal_copy_job_dir_output.py b/ush/subseasonal/subseasonal_copy_job_dir_output.py index 299d553576..7d72c04be8 100644 --- a/ush/subseasonal/subseasonal_copy_job_dir_output.py +++ b/ush/subseasonal/subseasonal_copy_job_dir_output.py @@ -19,8 +19,6 @@ STEP = os.environ['STEP'] COMPONENT = os.environ['COMPONENT'] JOB_GROUP = os.environ['JOB_GROUP'] -if STEP == 'plots': - NDAYS = os.environ['NDAYS'] # Copy files to desired location if STEP == 'stats': @@ -44,28 +42,24 @@ if JOB_GROUP == 'condense_stats': job_wildcard_dir = os.path.join( job_work_JOB_GROUP_dir, 'job*', f"{RUN}.*", '*', - f"last{NDAYS}days", '*', '*', '*', 'condensed_stats*' + '*', 'model1_*.stat' ) elif JOB_GROUP == 'filter_stats': job_wildcard_dir = os.path.join( job_work_JOB_GROUP_dir, 'job*', f"{RUN}.*", '*', - f"last{NDAYS}days", '*', '*', '*', 'fcst*.stat' + '*', 'fcst*.stat' ) elif JOB_GROUP == 'make_plots': job_wildcard_dir = os.path.join( job_work_JOB_GROUP_dir, 'job*', f"{RUN}.*", '*', - f"last{NDAYS}days", '*', '*', '*', '*', '*.png' + '*', '*', '*.png' + ) + elif JOB_GROUP == 'tar_images': + job_wildcard_dir = os.path.join( + job_work_JOB_GROUP_dir, 'job*', f"{RUN}.*", '*', + '*', '*.tar' ) output_file_JOB_list = glob.glob(job_wildcard_dir) -if STEP == 'plots' and JOB_GROUP == 'make_plots': - job_wildcard_dir2 = os.path.join( - job_work_JOB_GROUP_dir, 'job*', f"{RUN}.*", '*', - f"last{NDAYS}days", '*', '*', '*', '*', '*.gif' - ) - output_file_JOB_list = ( - output_file_JOB_list - + glob.glob(job_wildcard_dir2) - ) for output_file_JOB in sorted(output_file_JOB_list, key=len): output_file_end_path = output_file_JOB.partition( job_work_JOB_GROUP_dir+'/' From e1114dae0235bc17700d0764c1154f284bbe7387 Mon Sep 17 00:00:00 2001 From: shannon shields Date: Wed, 20 Nov 2024 18:16:32 +0000 Subject: [PATCH 4/9] Edit plots_job_scripts.py for MPMD output dir changes --- ...onal_plots_grid2grid_create_job_scripts.py | 40 ++++++++++++++++--- ...sonal_plots_grid2obs_create_job_scripts.py | 40 ++++++++++++++++--- 2 files changed, 68 insertions(+), 12 deletions(-) diff --git a/ush/subseasonal/subseasonal_plots_grid2grid_create_job_scripts.py b/ush/subseasonal/subseasonal_plots_grid2grid_create_job_scripts.py index b5ace06a6a..bbad355eab 100644 --- a/ush/subseasonal/subseasonal_plots_grid2grid_create_job_scripts.py +++ b/ush/subseasonal/subseasonal_plots_grid2grid_create_job_scripts.py @@ -554,13 +554,14 @@ +verif_type_job+'/' +job_env_dict['vx_mask'] ) - job_output_dir = os.path.join( + job_DATA_dir = os.path.join( DATA, VERIF_CASE+'_'+STEP, 'plot_output', RUN+'.'+end_date, verif_type, job_env_dict['job_name'].replace('/','_') ) - if not os.path.exists(job_output_dir): - os.makedirs(job_output_dir) + job_env_dict['job_DATA_dir'] = job_DATA_dir + if not os.path.exists(job_DATA_dir): + os.makedirs(job_DATA_dir) if JOB_GROUP == 'condense_stats': for JOB_GROUP_loop in list( itertools.product(model_list, [fcst_var_levels], @@ -598,10 +599,18 @@ # Set any environment variables for special cases # Write environment variables job_env_dict['job_id'] = 'job'+str(njobs) + job_env_dict['job_work_dir'] = os.path.join( + DATA, VERIF_CASE+'_'+STEP, 'plot_output', + 'job_work_dir', JOB_GROUP, + f"{job_env_dict['job_id']}", RUN+'.'+end_date, + verif_type, job_env_dict['job_name'].replace('/','_') + ) for name, value in job_env_dict.items(): job.write('export '+name+'='+value+'\n') job.write('\n') # Write job commands + if not os.path.exists(job_env_dict['job_work_dir']): + os.makedirs(job_env_dict['job_work_dir']) job.write( sub_util.python_command('subseasonal_plots.py',[]) +'\n' @@ -650,9 +659,17 @@ job.write('\n') # Write environment variables job_env_dict['job_id'] = 'job'+str(njobs) + job_env_dict['job_work_dir'] = os.path.join( + DATA, VERIF_CASE+'_'+STEP, 'plot_output', + 'job_work_dir', JOB_GROUP, + f"{job_env_dict['job_id']}", RUN+'.'+end_date, + verif_type, job_env_dict['job_name'].replace('/','_') + ) for name, value in job_env_dict.items(): job.write('export '+name+'='+value+'\n') job.write('\n') + if not os.path.exists(job_env_dict['job_work_dir']): + os.makedirs(job_env_dict['job_work_dir']) job.write( sub_util.python_command('subseasonal_plots.py',[]) +'\n' @@ -660,13 +677,14 @@ job.write('export err=$?; err_chk'+'\n') job.close() elif JOB_GROUP == 'make_plots': - job_output_images_dir = os.path.join( + job_DATA_images_dir = os.path.join( DATA, VERIF_CASE+'_'+STEP, 'plot_output', RUN+'.'+end_date, verif_type, job_env_dict['job_name'].replace('/','_'), 'images' ) - if not os.path.exists(job_output_images_dir): - os.makedirs(job_output_images_dir) + job_env_dict['job_DATA_images_dir'] = job_DATA_images_dir + if not os.path.exists(job_DATA_images_dir): + os.makedirs(job_DATA_images_dir) job_env_dict['model_list'] = "'"+f"{', '.join(model_list)}"+"'" job_env_dict['model_plot_name_list'] = ( "'"+f"{', '.join(model_plot_name_list)}"+"'" @@ -756,9 +774,19 @@ job.write('\n') # Write environment variables job_env_dict['job_id'] = 'job'+str(njobs) + job_env_dict['job_work_images_dir'] = os.path.join( + DATA, VERIF_CASE+'_'+STEP, 'plot_output', + 'job_work_dir', JOB_GROUP, + f"{job_env_dict['job_id']}", RUN+'.'+end_date, + verif_type, + job_env_dict['job_name'].replace('/','_'), + 'images' + ) for name, value in job_env_dict.items(): job.write('export '+name+'='+value+'\n') job.write('\n') + if not os.path.exists(job_env_dict['job_work_images_dir']): + os.makedirs(job_env_dict['job_work_images_dir']) job.write( sub_util.python_command(run_subseasonal_plot, [])+'\n' diff --git a/ush/subseasonal/subseasonal_plots_grid2obs_create_job_scripts.py b/ush/subseasonal/subseasonal_plots_grid2obs_create_job_scripts.py index aaf38c9974..2725b22544 100644 --- a/ush/subseasonal/subseasonal_plots_grid2obs_create_job_scripts.py +++ b/ush/subseasonal/subseasonal_plots_grid2obs_create_job_scripts.py @@ -258,13 +258,14 @@ +verif_type_job+'/' +job_env_dict['vx_mask'] ) - job_output_dir = os.path.join( + job_DATA_dir = os.path.join( DATA, VERIF_CASE+'_'+STEP, 'plot_output', RUN+'.'+end_date, verif_type, job_env_dict['job_name'].replace('/','_') ) - if not os.path.exists(job_output_dir): - os.makedirs(job_output_dir) + job_env_dict['job_DATA_dir'] = job_DATA_dir + if not os.path.exists(job_DATA_dir): + os.makedirs(job_DATA_dir) if JOB_GROUP == 'condense_stats': for JOB_GROUP_loop in list( itertools.product(model_list, [fcst_var_levels], @@ -302,10 +303,18 @@ # Set any environment variables for special cases # Write environment variables job_env_dict['job_id'] = 'job'+str(njobs) + job_env_dict['job_work_dir'] = os.path.join( + DATA, VERIF_CASE+'_'+STEP, 'plot_output', + 'job_work_dir', JOB_GROUP, + f"{job_env_dict['job_id']}", RUN+'.'+end_date, + verif_type, job_env_dict['job_name'].replace('/','_') + ) for name, value in job_env_dict.items(): job.write('export '+name+'='+value+'\n') job.write('\n') # Write job commands + if not os.path.exists(job_env_dict['job_work_dir']): + os.makedirs(job_env_dict['job_work_dir']) job.write( sub_util.python_command('subseasonal_plots.py',[]) +'\n' @@ -354,9 +363,17 @@ job.write('\n') # Write environment variables job_env_dict['job_id'] = 'job'+str(njobs) + job_env_dict['job_work_dir'] = os.path.join( + DATA, VERIF_CASE+'_'+STEP, 'plot_output', + 'job_work_dir', JOB_GROUP, + f"{job_env_dict['job_id']}", RUN+'.'+end_date, + verif_type, job_env_dict['job_name'].replace('/','_') + ) for name, value in job_env_dict.items(): job.write('export '+name+'='+value+'\n') job.write('\n') + if not os.path.exists(job_env_dict['job_work_dir']): + os.makedirs(job_env_dict['job_work_dir']) job.write( sub_util.python_command('subseasonal_plots.py',[]) +'\n' @@ -364,13 +381,14 @@ job.write('export err=$?; err_chk'+'\n') job.close() elif JOB_GROUP == 'make_plots': - job_output_images_dir = os.path.join( + job_DATA_images_dir = os.path.join( DATA, VERIF_CASE+'_'+STEP, 'plot_output', RUN+'.'+end_date, verif_type, job_env_dict['job_name'].replace('/','_'), 'images' ) - if not os.path.exists(job_output_images_dir): - os.makedirs(job_output_images_dir) + job_env_dict['job_DATA_images_dir'] = job_DATA_images_dir + if not os.path.exists(job_DATA_images_dir): + os.makedirs(job_DATA_images_dir) job_env_dict['model_list'] = "'"+f"{', '.join(model_list)}"+"'" job_env_dict['model_plot_name_list'] = ( "'"+f"{', '.join(model_plot_name_list)}"+"'" @@ -460,9 +478,19 @@ job.write('\n') # Write environment variables job_env_dict['job_id'] = 'job'+str(njobs) + job_env_dict['job_work_images_dir'] = os.path.join( + DATA, VERIF_CASE+'_'+STEP, 'plot_output', + 'job_work_dir', JOB_GROUP, + f"{job_env_dict['job_id']}", RUN+'.'+end_date, + verif_type, + job_env_dict['job_name'].replace('/','_'), + 'images' + ) for name, value in job_env_dict.items(): job.write('export '+name+'='+value+'\n') job.write('\n') + if not os.path.exists(job_env_dict['job_work_images_dir']): + os.makedirs(job_env_dict['job_work_images_dir']) job.write( sub_util.python_command(run_subseasonal_plot, [])+'\n' From c6fbfe652dd73145b831aaf8a74b4bddd962a8e0 Mon Sep 17 00:00:00 2001 From: shannon shields Date: Wed, 20 Nov 2024 21:39:36 +0000 Subject: [PATCH 5/9] Continue making MPMD output dir changes for plot scripts --- ush/subseasonal/subseasonal_plots.py | 88 ++++++++++--------- ...onal_plots_grid2grid_create_job_scripts.py | 7 ++ ...sonal_plots_grid2obs_create_job_scripts.py | 7 ++ 3 files changed, 62 insertions(+), 40 deletions(-) diff --git a/ush/subseasonal/subseasonal_plots.py b/ush/subseasonal/subseasonal_plots.py index bf642f7a1d..a9898d2699 100644 --- a/ush/subseasonal/subseasonal_plots.py +++ b/ush/subseasonal/subseasonal_plots.py @@ -64,7 +64,11 @@ fhr_end = os.environ['fhr_end'] fhr_inc = os.environ['fhr_inc'] job_id = os.environ['job_id'] +job_DATA_dir = os.environ['job_DATA_dir'] +job_work_dir = os.environ['job_work_dir'] if JOB_GROUP == 'make_plots': + job_DATA_images_dir = os.environ['job_DATA_images_dir'] + job_work_images_dir = os.environ['job_work_images_dir'] stat = os.environ['stat'] plot = os.environ['plot'] @@ -79,13 +83,13 @@ VERIF_CASE_STEP_dir = os.path.join(DATA, VERIF_CASE_STEP) stat_base_dir = os.path.join(VERIF_CASE_STEP_dir, 'data') plot_output_dir = os.path.join(VERIF_CASE_STEP_dir, 'plot_output') -logging_dir = os.path.join(plot_output_dir, RUN+'.'+end_date, 'logs') -VERIF_TYPE_image_dir = os.path.join(plot_output_dir, RUN+'.'+end_date, - 'images', VERIF_TYPE) -job_output_dir = os.path.join(plot_output_dir, RUN+'.'+end_date, - VERIF_TYPE, job_name.replace('/','_')) -if not os.path.exists(job_output_dir): - os.makedirs(job_output_dir) +logging_dir = os.path.join(job_work_dir, 'logs') +VERIF_TYPE_tar_dir = os.path.join(job_work_dir, RUN+'.'+end_date, + 'tar_files', VERIF_TYPE) +#job_output_dir = os.path.join(plot_output_dir, RUN+'.'+end_date, + #VERIF_TYPE, job_name.replace('/','_')) +#if not os.path.exists(job_output_dir): + #os.makedirs(job_output_dir) # Set up logging job_logging_file = os.path.join(logging_dir, 'evs_'+COMPONENT+'_'+RUN+'_' @@ -172,13 +176,14 @@ for model_idx in range(len(model_list)): model = model_list[model_idx] obs_name = obs_list[model_idx] - condensed_model_stat_file = os.path.join(job_output_dir, 'model' + condensed_model_stat_file = os.path.join(job_work_dir, 'model' +str(model_idx+1)+'_'+model +'.stat') sub_util.condense_model_stat_files(logger, stat_base_dir, condensed_model_stat_file, model, obs_name, grid, vx_mask, - fcst_var_name, obs_var_name, line_type) + fcst_var_name, obs_var_name, + line_type) elif JOB_GROUP == 'filter_stats': logger.info("Filtering model .stat files") model_info_dict = original_model_info_dict.copy() @@ -241,7 +246,7 @@ else: plot_dates = valid_dates all_model_df = sub_util.build_df( - logger, job_output_dir, job_output_dir, + logger, job_DATA_dir, job_work_dir, model_info_dict, met_info_dict, plot_info_dict['fcst_var_name'], plot_info_dict['fcst_var_level'], @@ -292,7 +297,7 @@ int(date_info_dict['forecast_hour']) ) image_name = plot_specs.get_savefig_name( - os.path.join(job_output_dir, 'images'), + job_work_images_dir, plot_info_dict, date_info_dict ) if init_hr in init_hrs: @@ -303,8 +308,9 @@ else: make_ts = False if make_ts: - plot_ts = subp_ts.TimeSeries(logger, job_output_dir, - job_output_dir, model_info_dict, + plot_ts = subp_ts.TimeSeries(logger, job_DATA_dir, + job_work_images_dir, + model_info_dict, date_info_dict, plot_info_dict, met_info_dict, logo_dir) plot_ts.make_time_series() @@ -325,7 +331,7 @@ plot_info_dict['obs_var_thresh'] = la_info[1][1][2] plot_info_dict['interp_points'] = str(la_info[2]) image_name = plot_specs.get_savefig_name( - os.path.join(job_output_dir, 'images'), + job_work_images_dir, plot_info_dict, date_info_dict ) if not os.path.exists(image_name): @@ -342,8 +348,9 @@ else: make_la = False if make_la: - plot_la = subp_la.LeadAverage(logger, job_output_dir, - job_output_dir, model_info_dict, + plot_la = subp_la.LeadAverage(logger, job_DATA_dir, + job_work_images_dir, + model_info_dict, date_info_dict, plot_info_dict, met_info_dict, logo_dir) plot_la.make_lead_average() @@ -363,7 +370,7 @@ plot_info_dict['obs_var_thresh'] = vha_info[0][1][2] plot_info_dict['interp_points'] = str(vha_info[1]) image_name = plot_specs.get_savefig_name( - os.path.join(job_output_dir, 'images'), + job_work_images_dir, plot_info_dict, date_info_dict ) if not os.path.exists(image_name): @@ -382,8 +389,8 @@ else: make_vha = False if make_vha: - plot_vha = subp_vha.ValidHourAverage(logger, job_output_dir, - job_output_dir, + plot_vha = subp_vha.ValidHourAverage(logger, job_DATA_dir, + job_work_images_dir, model_info_dict, date_info_dict, plot_info_dict, @@ -411,7 +418,7 @@ plot_info_dict['fcst_var_level'] = fcst_var_level_list[l] plot_info_dict['obs_var_level'] = obs_var_level_list[l] image_name = plot_specs.get_savefig_name( - os.path.join(job_output_dir, 'images'), + job_work_images_dir, plot_info_dict, date_info_dict ) if init_hr in init_hrs: @@ -431,8 +438,8 @@ else: make_ta = False if make_ta: - plot_ta = subp_ta.ThresholdAverage(logger, job_output_dir, - job_output_dir, + plot_ta = subp_ta.ThresholdAverage(logger, job_DATA_dir, + job_work_images_dir, model_info_dict, date_info_dict, plot_info_dict, @@ -456,7 +463,7 @@ plot_info_dict['obs_var_thresh'] = lbd_info[1][1][2] plot_info_dict['interp_points'] = str(lbd_info[2]) image_name = plot_specs.get_savefig_name( - os.path.join(job_output_dir, 'images'), + job_work_images_dir, plot_info_dict, date_info_dict ) if not os.path.exists(image_name): @@ -473,8 +480,9 @@ else: make_lbd = False if make_lbd: - plot_lbd = subp_lbd.LeadByDate(logger, job_output_dir, - job_output_dir, model_info_dict, + plot_lbd = subp_lbd.LeadByDate(logger, job_DATA_dir, + job_work_images_dir, + model_info_dict, date_info_dict, plot_info_dict, met_info_dict, logo_dir) plot_lbd.make_lead_by_date() @@ -502,7 +510,7 @@ plot_info_dict['fcst_var_thresh'] = fcst_var_thresh_list[t] plot_info_dict['obs_var_thresh'] = obs_var_thresh_list[t] image_name = plot_specs.get_savefig_name( - os.path.join(job_output_dir, 'images'), + job_work_images_dir, plot_info_dict, date_info_dict ) if init_hr in init_hrs: @@ -518,8 +526,8 @@ del plot_info_dict['fcst_var_level'] del plot_info_dict['obs_var_level'] if make_sbl: - plot_sbl = subp_sbl.StatByLevel(logger, job_output_dir, - job_output_dir, + plot_sbl = subp_sbl.StatByLevel(logger, job_DATA_dir, + job_work_images_dir, model_info_dict, date_info_dict, plot_info_dict, @@ -552,7 +560,7 @@ plot_info_dict['fcst_var_thresh'] = fcst_var_thresh_list[t] plot_info_dict['obs_var_thresh'] = obs_var_thresh_list[t] image_name = plot_specs.get_savefig_name( - os.path.join(job_output_dir, 'images'), + job_work_images_dir, plot_info_dict, date_info_dict ) if not os.path.exists(image_name): @@ -570,8 +578,8 @@ del plot_info_dict['fcst_var_level'] del plot_info_dict['obs_var_level'] if make_lbl: - plot_lbl = subp_lbl.LeadByLevel(logger, job_output_dir, - job_output_dir, + plot_lbl = subp_lbl.LeadByLevel(logger, job_DATA_dir, + job_work_images_dir, model_info_dict, date_info_dict, plot_info_dict, @@ -599,7 +607,7 @@ plot_info_dict['fcst_var_level'] = fcst_var_level_list[l] plot_info_dict['obs_var_level'] = obs_var_level_list[l] image_name = plot_specs.get_savefig_name( - os.path.join(job_output_dir, 'images'), + job_work_images_dir, plot_info_dict, date_info_dict ) if init_hr in init_hrs: @@ -610,8 +618,8 @@ else: make_pd = False if make_pd: - plot_pd = subp_pd.PerformanceDiagram(logger, job_output_dir, - job_output_dir, + plot_pd = subp_pd.PerformanceDiagram(logger, job_DATA_dir, + job_work_images_dir, model_info_dict, date_info_dict, plot_info_dict, @@ -624,21 +632,21 @@ # Create tar file of plots and move to main image directory elif JOB_GROUP == 'tar_images': logger.info("Tar'ing up images") - job_output_image_dir = os.path.join(job_output_dir, 'images') + job_input_image_dir = job_DATA_images_dir cwd = os.getcwd() - if len(glob.glob(job_output_image_dir+'/*')) != 0: - os.chdir(job_output_image_dir) - tar_file = os.path.join(VERIF_TYPE_image_dir, + if len(glob.glob(job_input_image_dir+'/*')) != 0: + os.chdir(job_input_image_dir) + tar_file = os.path.join(VERIF_TYPE_tar_dir, job_name.replace('/','_')+'.tar') if os.path.exists(tar_file): os.remove(tar_file) - logger.debug(f"Making tar file {tar_file} from {job_output_image_dir}") + logger.debug(f"Making tar file {tar_file} from {job_input_image_dir}") sub_util.run_shell_command( ['tar', '-cvf', tar_file, '*'] ) os.chdir(cwd) else: - logger.warning(f"No images generated in {job_output_image_dir}") + logger.warning(f"No images generated in {job_input_image_dir}") print("END: "+os.path.basename(__file__)) diff --git a/ush/subseasonal/subseasonal_plots_grid2grid_create_job_scripts.py b/ush/subseasonal/subseasonal_plots_grid2grid_create_job_scripts.py index bbad355eab..2c85de09e3 100644 --- a/ush/subseasonal/subseasonal_plots_grid2grid_create_job_scripts.py +++ b/ush/subseasonal/subseasonal_plots_grid2grid_create_job_scripts.py @@ -825,9 +825,16 @@ job.write('\n') # Write environment variables job_env_dict['job_id'] = 'job'+str(njobs) + job_env_dict['job_work_dir'] = os.path.join( + DATA, VERIF_CASE+'_'+STEP, 'plot_output', + 'job_work_dir', JOB_GROUP, + f"{job_env_dict['job_id']}" + ) for name, value in job_env_dict.items(): job.write('export '+name+'='+value+'\n') job.write('\n') + if not os.path.exists(job_env_dict['job_work_dir']): + os.makedirs(job_env_dict['job_work_dir']) job.write( sub_util.python_command('subseasonal_plots.py',[]) +'\n' diff --git a/ush/subseasonal/subseasonal_plots_grid2obs_create_job_scripts.py b/ush/subseasonal/subseasonal_plots_grid2obs_create_job_scripts.py index 2725b22544..46a6145e9c 100644 --- a/ush/subseasonal/subseasonal_plots_grid2obs_create_job_scripts.py +++ b/ush/subseasonal/subseasonal_plots_grid2obs_create_job_scripts.py @@ -529,9 +529,16 @@ job.write('\n') # Write environment variables job_env_dict['job_id'] = 'job'+str(njobs) + job_env_dict['job_work_dir'] = os.path.join( + DATA, VERIF_CASE+'_'+STEP, 'plot_output', + 'job_work_dir', JOB_GROUP, + f"{job_env_dict['job_id']}" + ) for name, value in job_env_dict.items(): job.write('export '+name+'='+value+'\n') job.write('\n') + if not os.path.exists(job_env_dict['job_work_dir']): + os.makedirs(job_env_dict['job_work_dir']) job.write( sub_util.python_command('subseasonal_plots.py',[]) +'\n' From c37f10d2ca31dfa10c09bce45d7239f7960eae1e Mon Sep 17 00:00:00 2001 From: shannon shields Date: Thu, 21 Nov 2024 16:06:44 +0000 Subject: [PATCH 6/9] Finish making changes for MPMD plots bugzilla --- .../subseasonal/exevs_subseasonal_grid2grid_plots.sh | 4 ++-- .../subseasonal/exevs_subseasonal_grid2obs_plots.sh | 4 ++-- ush/subseasonal/subseasonal_plots.py | 9 +++++++-- .../subseasonal_plots_grid2grid_create_job_scripts.py | 7 +++++++ .../subseasonal_plots_grid2obs_create_job_scripts.py | 7 +++++++ ush/subseasonal/subseasonal_plots_lead_average.py | 2 +- ush/subseasonal/subseasonal_plots_performance_diagram.py | 2 +- ush/subseasonal/subseasonal_plots_time_series.py | 2 +- 8 files changed, 28 insertions(+), 9 deletions(-) diff --git a/scripts/plots/subseasonal/exevs_subseasonal_grid2grid_plots.sh b/scripts/plots/subseasonal/exevs_subseasonal_grid2grid_plots.sh index 461891e370..a93926693a 100755 --- a/scripts/plots/subseasonal/exevs_subseasonal_grid2grid_plots.sh +++ b/scripts/plots/subseasonal/exevs_subseasonal_grid2grid_plots.sh @@ -79,9 +79,9 @@ for group in condense_stats filter_stats make_plots tar_images; do export err=$?; err_chk # Cat the plotting log files if [ $JOB_GROUP = make_plots ]; then - log_dir=$DATA/${VERIF_CASE}_${STEP}/plot_output/job_work_dir/${JOB_GROUP}/job*/*/*/*/*/*/*/*/logs + log_dir=$DATA/${VERIF_CASE}_${STEP}/plot_output/job_work_dir/${JOB_GROUP}/job*/*/*/*/*/logs else - log_dir=$DATA/${VERIF_CASE}_${STEP}/plot_output/job_work_dir/${JOB_GROUP}/job*/*/*/*/*/*/*/logs + log_dir=$DATA/${VERIF_CASE}_${STEP}/plot_output/job_work_dir/${JOB_GROUP}/job*/*/*/*/logs fi log_file_count=$(find $log_dir -type f |wc -l) if [[ $log_file_count -ne 0 ]]; then diff --git a/scripts/plots/subseasonal/exevs_subseasonal_grid2obs_plots.sh b/scripts/plots/subseasonal/exevs_subseasonal_grid2obs_plots.sh index f608085227..e2175fd31f 100755 --- a/scripts/plots/subseasonal/exevs_subseasonal_grid2obs_plots.sh +++ b/scripts/plots/subseasonal/exevs_subseasonal_grid2obs_plots.sh @@ -79,9 +79,9 @@ for group in condense_stats filter_stats make_plots tar_images; do export err=$?; err_chk # Cat the plotting log files if [ $JOB_GROUP = make_plots ]; then - log_dir=$DATA/${VERIF_CASE}_${STEP}/plot_output/job_work_dir/${JOB_GROUP}/job*/*/*/*/*/*/*/*/logs + log_dir=$DATA/${VERIF_CASE}_${STEP}/plot_output/job_work_dir/${JOB_GROUP}/job*/*/*/*/*/logs else - log_dir=$DATA/${VERIF_CASE}_${STEP}/plot_output/job_work_dir/${JOB_GROUP}/job*/*/*/*/*/*/*/logs + log_dir=$DATA/${VERIF_CASE}_${STEP}/plot_output/job_work_dir/${JOB_GROUP}/job*/*/*/*/logs fi log_file_count=$(find $log_dir -type f |wc -l) if [[ $log_file_count -ne 0 ]]; then diff --git a/ush/subseasonal/subseasonal_plots.py b/ush/subseasonal/subseasonal_plots.py index a9898d2699..fab21616a8 100644 --- a/ush/subseasonal/subseasonal_plots.py +++ b/ush/subseasonal/subseasonal_plots.py @@ -82,10 +82,15 @@ logo_dir = os.path.join(FIXevs, 'logos') VERIF_CASE_STEP_dir = os.path.join(DATA, VERIF_CASE_STEP) stat_base_dir = os.path.join(VERIF_CASE_STEP_dir, 'data') -plot_output_dir = os.path.join(VERIF_CASE_STEP_dir, 'plot_output') -logging_dir = os.path.join(job_work_dir, 'logs') +#plot_output_dir = os.path.join(VERIF_CASE_STEP_dir, 'plot_output') VERIF_TYPE_tar_dir = os.path.join(job_work_dir, RUN+'.'+end_date, 'tar_files', VERIF_TYPE) +if JOB_GROUP == 'make_plots': + logging_dir = os.path.join(job_work_images_dir, 'logs') +elif JOB_GROUP == 'tar_images': + logging_dir = os.path.join(VERIF_TYPE_tar_dir, 'logs') +else: + logging_dir = os.path.join(job_work_dir, 'logs') #job_output_dir = os.path.join(plot_output_dir, RUN+'.'+end_date, #VERIF_TYPE, job_name.replace('/','_')) #if not os.path.exists(job_output_dir): diff --git a/ush/subseasonal/subseasonal_plots_grid2grid_create_job_scripts.py b/ush/subseasonal/subseasonal_plots_grid2grid_create_job_scripts.py index 2c85de09e3..bf05b5dd26 100644 --- a/ush/subseasonal/subseasonal_plots_grid2grid_create_job_scripts.py +++ b/ush/subseasonal/subseasonal_plots_grid2grid_create_job_scripts.py @@ -794,6 +794,13 @@ job.write('export err=$?; err_chk'+'\n') job.close() elif JOB_GROUP == 'tar_images': + job_DATA_dir = os.path.join( + DATA, VERIF_CASE+'_'+STEP, 'plot_output', + RUN+'.'+end_date, 'tar_files', verif_type + ) + job_env_dict['job_DATA_dir'] = job_DATA_dir + if not os.path.exists(job_env_dict['job_DATA_dir']): + os.makedirs(job_env_dict['job_DATA_dir']) job_env_dict['model_list'] = "'"+f"{', '.join(model_list)}"+"'" job_env_dict['model_plot_name_list'] = ( "'"+f"{', '.join(model_plot_name_list)}"+"'" diff --git a/ush/subseasonal/subseasonal_plots_grid2obs_create_job_scripts.py b/ush/subseasonal/subseasonal_plots_grid2obs_create_job_scripts.py index 46a6145e9c..108b3d1aed 100644 --- a/ush/subseasonal/subseasonal_plots_grid2obs_create_job_scripts.py +++ b/ush/subseasonal/subseasonal_plots_grid2obs_create_job_scripts.py @@ -498,6 +498,13 @@ job.write('export err=$?; err_chk'+'\n') job.close() elif JOB_GROUP == 'tar_images': + job_DATA_dir = os.path.join( + DATA, VERIF_CASE+'_'+STEP, 'plot_output', + RUN+'.'+end_date, 'tar_files', verif_type + ) + job_env_dict['job_DATA_dir'] = job_DATA_dir + if not os.path.exists(job_env_dict['job_DATA_dir']): + os.makedirs(job_env_dict['job_DATA_dir']) job_env_dict['model_list'] = "'"+f"{', '.join(model_list)}"+"'" job_env_dict['model_plot_name_list'] = ( "'"+f"{', '.join(model_plot_name_list)}"+"'" diff --git a/ush/subseasonal/subseasonal_plots_lead_average.py b/ush/subseasonal/subseasonal_plots_lead_average.py index af4ad1dd19..f129fe5c45 100644 --- a/ush/subseasonal/subseasonal_plots_lead_average.py +++ b/ush/subseasonal/subseasonal_plots_lead_average.py @@ -74,7 +74,7 @@ def make_lead_average(self): +f"{self.plot_info_dict['stat']}") sys.exit(0) # Make job image directory - output_image_dir = os.path.join(self.output_dir, 'images') + output_image_dir = self.output_dir if not os.path.exists(output_image_dir): os.makedirs(output_image_dir) self.logger.info(f"Plots will be in: {output_image_dir}") diff --git a/ush/subseasonal/subseasonal_plots_performance_diagram.py b/ush/subseasonal/subseasonal_plots_performance_diagram.py index 5efd725f6c..37b7a10d38 100644 --- a/ush/subseasonal/subseasonal_plots_performance_diagram.py +++ b/ush/subseasonal/subseasonal_plots_performance_diagram.py @@ -74,7 +74,7 @@ def make_performance_diagram(self): +f"{self.plot_info_dict['stat']}") sys.exit(0) # Make job image directory - output_image_dir = os.path.join(self.output_dir, 'images') + output_image_dir = self.output_dir if not os.path.exists(output_image_dir): os.makedirs(output_image_dir) self.logger.info(f"Plots will be in: {output_image_dir}") diff --git a/ush/subseasonal/subseasonal_plots_time_series.py b/ush/subseasonal/subseasonal_plots_time_series.py index fb25e6576f..546e08a14d 100644 --- a/ush/subseasonal/subseasonal_plots_time_series.py +++ b/ush/subseasonal/subseasonal_plots_time_series.py @@ -69,7 +69,7 @@ def make_time_series(self): self.logger.debug(f"Plot information dictionary: " +f"{self.plot_info_dict}") # Make job image directory - output_image_dir = os.path.join(self.output_dir, 'images') + output_image_dir = self.output_dir if not os.path.exists(output_image_dir): os.makedirs(output_image_dir) self.logger.info(f"Plots will be in: {output_image_dir}") From 16b0e648e1205abdd13a1e57a5fc5c16746f7adb Mon Sep 17 00:00:00 2001 From: shannon shields Date: Thu, 21 Nov 2024 17:39:28 +0000 Subject: [PATCH 7/9] Make logging_dir in subseasonal_plots.py --- ush/subseasonal/subseasonal_plots.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ush/subseasonal/subseasonal_plots.py b/ush/subseasonal/subseasonal_plots.py index fab21616a8..8f156a648b 100644 --- a/ush/subseasonal/subseasonal_plots.py +++ b/ush/subseasonal/subseasonal_plots.py @@ -91,6 +91,8 @@ logging_dir = os.path.join(VERIF_TYPE_tar_dir, 'logs') else: logging_dir = os.path.join(job_work_dir, 'logs') +if not os.path.exists(logging_dir): + os.makedirs(logging_dir) #job_output_dir = os.path.join(plot_output_dir, RUN+'.'+end_date, #VERIF_TYPE, job_name.replace('/','_')) #if not os.path.exists(job_output_dir): From 41fa1e45c34b104db5d3dc3c0befe536f4e798b8 Mon Sep 17 00:00:00 2001 From: shannon shields Date: Thu, 21 Nov 2024 20:09:27 +0000 Subject: [PATCH 8/9] Make fixes in subseasonal_plots.py --- ush/subseasonal/subseasonal_plots.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/ush/subseasonal/subseasonal_plots.py b/ush/subseasonal/subseasonal_plots.py index 8f156a648b..77524a74da 100644 --- a/ush/subseasonal/subseasonal_plots.py +++ b/ush/subseasonal/subseasonal_plots.py @@ -65,12 +65,13 @@ fhr_inc = os.environ['fhr_inc'] job_id = os.environ['job_id'] job_DATA_dir = os.environ['job_DATA_dir'] -job_work_dir = os.environ['job_work_dir'] if JOB_GROUP == 'make_plots': job_DATA_images_dir = os.environ['job_DATA_images_dir'] job_work_images_dir = os.environ['job_work_images_dir'] stat = os.environ['stat'] plot = os.environ['plot'] +else: + job_work_dir = os.environ['job_work_dir'] # Set variables VERIF_CASE_STEP = VERIF_CASE+'_'+STEP @@ -83,8 +84,9 @@ VERIF_CASE_STEP_dir = os.path.join(DATA, VERIF_CASE_STEP) stat_base_dir = os.path.join(VERIF_CASE_STEP_dir, 'data') #plot_output_dir = os.path.join(VERIF_CASE_STEP_dir, 'plot_output') -VERIF_TYPE_tar_dir = os.path.join(job_work_dir, RUN+'.'+end_date, - 'tar_files', VERIF_TYPE) +if JOB_GROUP == 'tar_images': + VERIF_TYPE_tar_dir = os.path.join(job_work_dir, RUN+'.'+end_date, + 'tar_files', VERIF_TYPE) if JOB_GROUP == 'make_plots': logging_dir = os.path.join(job_work_images_dir, 'logs') elif JOB_GROUP == 'tar_images': @@ -639,7 +641,11 @@ # Create tar file of plots and move to main image directory elif JOB_GROUP == 'tar_images': logger.info("Tar'ing up images") - job_input_image_dir = job_DATA_images_dir + job_input_image_dir = os.path.join( + DATA, VERIF_CASE+'_'+STEP, 'plot_output', + RUN+'.'+end_date, VERIF_TYPE, + job_name.replace('/','_'), 'images' + ) cwd = os.getcwd() if len(glob.glob(job_input_image_dir+'/*')) != 0: os.chdir(job_input_image_dir) From 042f8593aa4a9ff1b4789cdae1ea7948be6aa651 Mon Sep 17 00:00:00 2001 From: shannon shields Date: Fri, 22 Nov 2024 19:31:28 +0000 Subject: [PATCH 9/9] Remove old/commented out code --- .../subseasonal/exevs_subseasonal_grid2grid_plots.sh | 11 ----------- .../create_METplus_subseasonal_output_dirs.py | 5 ----- ush/subseasonal/subseasonal_plots.py | 5 ----- 3 files changed, 21 deletions(-) diff --git a/scripts/plots/subseasonal/exevs_subseasonal_grid2grid_plots.sh b/scripts/plots/subseasonal/exevs_subseasonal_grid2grid_plots.sh index a93926693a..e9cd656e59 100755 --- a/scripts/plots/subseasonal/exevs_subseasonal_grid2grid_plots.sh +++ b/scripts/plots/subseasonal/exevs_subseasonal_grid2grid_plots.sh @@ -93,17 +93,6 @@ for group in condense_stats filter_stats make_plots tar_images; do fi done -# Cat the plotting log files -#log_dir=$DATA/${VERIF_CASE}_${STEP}/plot_output/${RUN}.${end_date}/logs -#log_file_count=$(find $log_dir -type f |wc -l) -#if [[ $log_file_count -ne 0 ]]; then - #for log_file in $log_dir/*; do - #echo "Start: $log_file" - #cat $log_file - #echo "End: $log_file" - #done -#fi - # Copy files to desired location if [ $SENDCOM = YES ]; then # Make and copy tar file diff --git a/ush/subseasonal/create_METplus_subseasonal_output_dirs.py b/ush/subseasonal/create_METplus_subseasonal_output_dirs.py index 9ff085995d..6a3fa2f2ce 100644 --- a/ush/subseasonal/create_METplus_subseasonal_output_dirs.py +++ b/ush/subseasonal/create_METplus_subseasonal_output_dirs.py @@ -126,11 +126,6 @@ os.path.join(working_output_base_dir, RUN+'.'+end_date_dt.strftime('%Y%m%d')) ) - #working_dir_list.append( - #os.path.join(working_output_base_dir, - #RUN+'.'+end_date_dt.strftime('%Y%m%d'), - #'logs') - #) for VCS_type in VCS_type_list: working_dir_list.append( os.path.join(working_output_base_dir, diff --git a/ush/subseasonal/subseasonal_plots.py b/ush/subseasonal/subseasonal_plots.py index 77524a74da..84235f1db1 100644 --- a/ush/subseasonal/subseasonal_plots.py +++ b/ush/subseasonal/subseasonal_plots.py @@ -83,7 +83,6 @@ logo_dir = os.path.join(FIXevs, 'logos') VERIF_CASE_STEP_dir = os.path.join(DATA, VERIF_CASE_STEP) stat_base_dir = os.path.join(VERIF_CASE_STEP_dir, 'data') -#plot_output_dir = os.path.join(VERIF_CASE_STEP_dir, 'plot_output') if JOB_GROUP == 'tar_images': VERIF_TYPE_tar_dir = os.path.join(job_work_dir, RUN+'.'+end_date, 'tar_files', VERIF_TYPE) @@ -95,10 +94,6 @@ logging_dir = os.path.join(job_work_dir, 'logs') if not os.path.exists(logging_dir): os.makedirs(logging_dir) -#job_output_dir = os.path.join(plot_output_dir, RUN+'.'+end_date, - #VERIF_TYPE, job_name.replace('/','_')) -#if not os.path.exists(job_output_dir): - #os.makedirs(job_output_dir) # Set up logging job_logging_file = os.path.join(logging_dir, 'evs_'+COMPONENT+'_'+RUN+'_'