From a2ba32bc08fad479bfbac23cb40ccd80104b00e1 Mon Sep 17 00:00:00 2001 From: AliS-Noaa Date: Mon, 30 Dec 2024 18:42:48 +0000 Subject: [PATCH 01/22] optimized the ww3_outp, issue #3157 --- scripts/exgfs_wave_post_pnt.sh | 177 ++++++++++++++++++++++++++++----- 1 file changed, 150 insertions(+), 27 deletions(-) diff --git a/scripts/exgfs_wave_post_pnt.sh b/scripts/exgfs_wave_post_pnt.sh index 2355d8f51e..27d1375373 100755 --- a/scripts/exgfs_wave_post_pnt.sh +++ b/scripts/exgfs_wave_post_pnt.sh @@ -252,7 +252,7 @@ HMS="$(echo $CDATE | cut -c9-10)0000" if [ -f $COMIN/rundata/${WAV_MOD_TAG}.out_pnt.${waveuoutpGRD}.${YMD}.${HMS} ] then - ln -s $COMIN/rundata/${WAV_MOD_TAG}.out_pnt.${waveuoutpGRD}.${YMD}.${HMS} ./out_pnt.${waveuoutpGRD} + ln -s $COMIN/rundata/${WAV_MOD_TAG}.out_pnt.${waveuoutpGRD}.${YMD}.${HMS} else echo '*************************************************** ' echo " FATAL ERROR : NO RAW POINT OUTPUT FILE out_pnt.${waveuoutpGRD}.${YMD}.${HMS} " @@ -264,10 +264,10 @@ fi rm -f buoy_tmp.loc buoy_log.ww3 ww3_oup.inp - ln -fs ./out_pnt.${waveuoutpGRD} ./out_pnt.ww3 + ln -fs ${WAV_MOD_TAG}.out_pnt.${waveuoutpGRD}.${YMD}.${HMS} ./out_pnt.ww3 ln -fs ./mod_def.${waveuoutpGRD} ./mod_def.ww3 export pgm=ww3_outp;. prep_step - $EXECwave/ww3_outp > buoy_lst.loc 2>&1 + $EXECwave/ww3_outp ${WAV_MOD_TAG} > buoy_lst.loc 2>&1 export err=$?;err_chk @@ -345,6 +345,104 @@ echo ' Making command file for wave post points ' [[ "$LOUD" = YES ]] && set -x + if true; then + grep -F -f ibp_tags buoy_lst.loc | awk '{ print $1 }' > buoys + grep -F -f buoys buoy_log.ww3 | awk '{ print $1 }' > points + rm buoys + + if [ "$DOBLL_WAV" = "YES" ] && [ "$DOSPC_WAV" = "NO" ]; then + cat << EOF > ww3_outp_auto.inp +$ WAVEWATCH III Point output post-processing +$ ------------------------------------------ + TIME DT COUNT +$ +EOF + cat points >> ww3_outp_auto.inp + cat << EOF >> ww3_outp_auto.inp + -1 +$ + 4 + 4 1 REFT 'UTC' +$ +$ End of input file +EOF + + elif [ "$DOSPC_WAV" = "YES" ] && [ "$DOBLL_WAV" = "NO" ]; then + cat << EOF > ww3_outp_auto.inp +$ WAVEWATCH III Point output post-processing +$ ------------------------------------------ + TIME DT COUNT +$ +EOF + cat points >> ww3_outp_auto.inp + cat << EOF >> ww3_outp_auto.inp + -1 +$ + 1 + 3 0. 0. 51 FORMAT +$ +$ End of input file +EOF + + elif [ "$DOSPC_WAV" = "YES" ] && [ "$DOBLL_WAV" = "YES" ]; then + cat << EOF > ww3_outp_auto.inp +$ WAVEWATCH III Point output post-processing +$ ------------------------------------------ + TIME DT COUNT +$ +EOF + cat points >> ww3_outp_auto.inp + cat << EOF >> ww3_outp_auto.inp + -1 +$ + 1 + 3 0. 0. 51 FORMAT +$ +$ End of input file +EOF + + fi + + rm points + N=$(( ($FHMAX_WAV_PNT - $FHMIN_WAV) + 1 )) + tstart="`echo $ymdh | cut -c1-8` `echo $ymdh | cut -c9-10`0000" + truntime="`echo $CDATE | cut -c1-8` `echo $CDATE | cut -c9-10`0000" + sed -e "s/TIME/$tstart/g" \ + -e "s/DT/$dtspec/g" \ + -e "s/COUNT/$N/g" \ + -e "s/REFT/$truntime/g" ww3_outp_auto.inp > ww3_outp.inp + + cp $DATA/mod_def.${waveuoutpGRD} mod_def.${waveuoutpGRD} + + fhr=$FHMIN_WAV + while [ $fhr -le $FHMAX_WAV_PNT ]; do + + #echo " Creating the wave point scripts at : `date`" + ymdh=`$NDATE $fhr $CDATE` + YMD=$(echo $ymdh | cut -c1-8) + HMS="$(echo $ymdh | cut -c9-10)0000" + + pfile=$COMIN/rundata/${WAV_MOD_TAG}.out_pnt.${waveuoutpGRD}.${YMD}.${HMS} + if [ -f ${pfile} ] + then + ln -fs ${pfile} + else + echo " FATAL ERROR : NO RAW POINT OUTPUT FILE out_pnt.$waveuoutpGRD.${YMD}.${HMS} " + echo ' ' + [[ "$LOUD" = YES ]] && set -x + err=7; export err;${errchk} + exit $err + fi + FHINCP=$(( DTPNT_WAV / 3600 )) + fhrp=$((fhr+FHINCP)) + fhr=$fhrp # no gridded output, loop with out_pnt stride + done + + + $EXECwave/ww3_outp ${WAV_MOD_TAG} 1> ww3_outp.log 2>&1 + + + else rm -f cmdfile touch cmdfile chmod 744 cmdfile @@ -390,6 +488,7 @@ for buoy in $buoys do echo "$USHwave/wave_outp_spec.sh $buoy $ymdh spec $SPECDATA > $SPECDATA/spec_$buoy.out 2>&1" >> tmpcmdfile.$FH3 + #echo "strace -o strace.${buoy}.${ymdh} $USHwave/wave_outp_spec.sh $buoy $ymdh spec $SPECDATA > $SPECDATA/spec_$buoy.out 2>&1" >> tmpcmdfile.$FH3 done fi @@ -398,20 +497,30 @@ export dtspec=3600. for buoy in $buoys do - echo "$USHwave/wave_outp_spec.sh $buoy $ymdh bull $SPECDATA > $SPECDATA/bull_$buoy.out 2>&1" >> tmpcmdfile.$FH3 + #echo "$USHwave/wave_outp_spec.sh $buoy $ymdh bull $SPECDATA > $SPECDATA/bull_$buoy.out 2>&1" >> tmpcmdfile.$FH3 + #echo "LD_LIBRARY_PATH="/lib64:/usr/lib64:${LD_LIBRARY_PATH}" && $USHwave/wave_outp_spec.sh $buoy $ymdh bull $SPECDATA > $SPECDATA/bull_$buoy.out 2>&1" >> tmpcmdfile.$FH3 + #echo "/usr/bin/mkdir -p /tmp/output_$YMDHMS; $USHwave/wave_outp_spec.sh $buoy $ymdh bull /tmp/output_$YMDHMS > $SPECDATA/bull_$buoy.out 2>&1" >> tmpcmdfile.$FH3 + echo "/usr/bin/mkdir -p /tmp/output_$YMDHMS; LD_LIBRARY_PATH="/lib64:/usr/lib64:${LD_LIBRARY_PATH}" && $USHwave/wave_outp_spec.sh $buoy $ymdh bull /tmp/output_$YMDHMS > $SPECDATA/bull_$buoy.out 2>&1" >> tmpcmdfile.$FH3 + #echo "/usr/bin/mkdir -p /${SCRATCH}/output_$YMDHMS; LD_LIBRARY_PATH="/lib64:/usr/lib64:${LD_LIBRARY_PATH}" && $USHwave/wave_outp_spec.sh $buoy $ymdh bull /${SCRATCH}/output_$YMDHMS > $SPECDATA/bull_$buoy.out 2>&1" >> tmpcmdfile.$FH3 + #echo "/usr/bin/strace -o strace.${buoy}.${ymdh} $USHwave/wave_outp_spec.sh $buoy $ymdh bull $SPECDATA > $SPECDATA/bull_$buoy.out 2>&1" >> tmpcmdfile.$FH3 done fi - split -n l/1/10 tmpcmdfile.$FH3 > cmdfile.${FH3}.01 - split -n l/2/10 tmpcmdfile.$FH3 > cmdfile.${FH3}.02 - split -n l/3/10 tmpcmdfile.$FH3 > cmdfile.${FH3}.03 - split -n l/4/10 tmpcmdfile.$FH3 > cmdfile.${FH3}.04 - split -n l/5/10 tmpcmdfile.$FH3 > cmdfile.${FH3}.05 - split -n l/6/10 tmpcmdfile.$FH3 > cmdfile.${FH3}.06 - split -n l/7/10 tmpcmdfile.$FH3 > cmdfile.${FH3}.07 - split -n l/8/10 tmpcmdfile.$FH3 > cmdfile.${FH3}.08 - split -n l/9/10 tmpcmdfile.$FH3 > cmdfile.${FH3}.09 - split -n l/10/10 tmpcmdfile.$FH3 > cmdfile.${FH3}.10 + for n in $(seq -w 1 10) + do + echo "export LD_LIBRARY_PATH=\"/lib64:/usr/lib64:${LD_LIBRARY_PATH}\"" > cmdfile.${FH3}.${n} + done + + split -n l/1/10 tmpcmdfile.$FH3 >> cmdfile.${FH3}.01 + split -n l/2/10 tmpcmdfile.$FH3 >> cmdfile.${FH3}.02 + split -n l/3/10 tmpcmdfile.$FH3 >> cmdfile.${FH3}.03 + split -n l/4/10 tmpcmdfile.$FH3 >> cmdfile.${FH3}.04 + split -n l/5/10 tmpcmdfile.$FH3 >> cmdfile.${FH3}.05 + split -n l/6/10 tmpcmdfile.$FH3 >> cmdfile.${FH3}.06 + split -n l/7/10 tmpcmdfile.$FH3 >> cmdfile.${FH3}.07 + split -n l/8/10 tmpcmdfile.$FH3 >> cmdfile.${FH3}.08 + split -n l/9/10 tmpcmdfile.$FH3 >> cmdfile.${FH3}.09 + split -n l/10/10 tmpcmdfile.$FH3 >> cmdfile.${FH3}.10 rm tmpcmdfile.$FH3 chmod 744 cmdfile.${FH3}.01 cmdfile.${FH3}.02 cmdfile.${FH3}.03 cmdfile.${FH3}.04 @@ -449,7 +558,8 @@ else if [ "$ifirst" = 'yes' ]; then echo "#!/bin/sh" > cmdmfile.$nfile - echo "$nfile cmdmfile.$nfile" >> cmdmprog +# echo "$nfile cmdmfile.$nfile" >> cmdmprog + echo "$DATA/cmdmfile.$nfile" >> cmdmprog chmod 744 cmdmfile.$nfile fi echo $line >> cmdmfile.$nfile @@ -476,13 +586,16 @@ if [ "$wavenproc" -gt '1' ] then if [ ${CFP_MP:-"NO"} = "YES" ]; then - ${wavempexec} -n ${wavenproc} ${wave_mpmd} cmdmprog + echo "exgfs_wave_post_pnt: Running cmdmprog using ${wavempexec} on ${wavenproc}" + ${wavempexec} ${wave_mpmd} ${DATA}/cmdmprog else + echo "exgfs_wave_post_pnt: Running cmdfile using ${wavempexec} on ${wavenproc}. ${NTASKS}" ${wavempexec} ${wavenproc} ${wave_mpmd} cmdfile fi exit=$? else chmod 744 cmdfile + echo "exgfs_wave_post_pnt: Running cmdfile serially" ./cmdfile exit=$? fi @@ -524,7 +637,9 @@ then for buoy in $buoys do - echo "$USHwave/wave_outp_cat.sh $buoy $FHMAX_WAV_PNT bull > ${CATOUTDIR}/bull_cat_$buoy.out 2>&1" >> cmdfile.bouy + #echo "$USHwave/wave_outp_cat.sh $buoy $FHMAX_WAV_PNT bull > ${CATOUTDIR}/bull_cat_$buoy.out 2>&1" >> cmdfile.bouy + echo "LD_LIBRARY_PATH="/lib64:/usr/lib64:${LD_LIBRARY_PATH}" && $USHwave/wave_outp_cat.sh $buoy $FHMAX_WAV_PNT bull > ${CATOUTDIR}/bull_cat_$buoy.out 2>&1" >> cmdfile.bouy + #echo "LD_LIBRARY_PATH="/lib64:/usr/lib64:${LD_LIBRARY_PATH}" && /usr/bin/strace -o strace.${buoy}.${ymdh}.cat $USHwave/wave_outp_cat.sh $buoy $FHMAX_WAV_PNT bull > ${CATOUTDIR}/bull_cat_$buoy.out 2>&1" >> cmdfile.bouy done fi @@ -541,7 +656,8 @@ else if [ "$ifirst" = 'yes' ]; then echo "#!/bin/sh" > cmdfile.bouy.$nfile - echo "$nfile cmdfile.bouy.$nfile" >> cmdmprogbouy +# echo "$nfile cmdfile.bouy.$nfile" >> cmdmprogbouy + echo "$DATA/cmdfile.bouy.$nfile" >> cmdmprogbouy chmod 744 cmdfile.bouy.$nfile fi echo $line >> cmdfile.bouy.$nfile @@ -568,7 +684,7 @@ if [ "$wavenproc" -gt '1' ] then if [ ${CFP_MP:-"NO"} = "YES" ]; then - ${wavempexec} -n ${wavenproc} ${wave_mpmd} cmdmprogbouy + ${wavempexec} ${wave_mpmd} ${DATA}/cmdmprogbouy else ${wavempexec} ${wavenproc} ${wave_mpmd} cmdfile.bouy fi @@ -576,6 +692,7 @@ else chmod 744 ${fcmdnow} ./${fcmdnow} + #./cmdfile.bouy exit=$? fi @@ -592,7 +709,7 @@ err=9; export err;${errchk} exit $err fi - + fi # dkokron New versus Old # --------------------------------------------------------------------------- # # 3. Compress point output data into tar files @@ -616,24 +733,30 @@ if [ ${CFP_MP:-"NO"} = "YES" ]; then if [ "$DOBNDPNT_WAV" = YES ]; then if [ "$DOSPC_WAV" = YES ]; then - echo "$nm $USHwave/wave_tar.sh $WAV_MOD_TAG ibp $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 " >> cmdtarfile +# echo "$nm $USHwave/wave_tar.sh $WAV_MOD_TAG ibp $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 " >> cmdtarfile + echo "$USHwave/wave_tar.sh $WAV_MOD_TAG ibp $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 " >> cmdtarfile nm=$(( nm + 1 )) fi if [ "$DOBLL_WAV" = YES ]; then - echo "$nm $USHwave/wave_tar.sh $WAV_MOD_TAG ibpbull $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 " >> cmdtarfile +# echo "$nm $USHwave/wave_tar.sh $WAV_MOD_TAG ibpbull $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 " >> cmdtarfile + echo "$USHwave/wave_tar.sh $WAV_MOD_TAG ibpbull $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 " >> cmdtarfile nm=$(( nm + 1 )) - echo "$nm $USHwave/wave_tar.sh $WAV_MOD_TAG ibpcbull $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 " >> cmdtarfile +# echo "$nm $USHwave/wave_tar.sh $WAV_MOD_TAG ibpcbull $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 " >> cmdtarfile + echo "$USHwave/wave_tar.sh $WAV_MOD_TAG ibpcbull $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 " >> cmdtarfile nm=$(( nm + 1 )) fi else if [ "$DOSPC_WAV" = YES ]; then - echo "$nm $USHwave/wave_tar.sh $WAV_MOD_TAG spec $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 " >> cmdtarfile +# echo "$nm $USHwave/wave_tar.sh $WAV_MOD_TAG spec $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 " >> cmdtarfile + echo "$USHwave/wave_tar.sh $WAV_MOD_TAG spec $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 " >> cmdtarfile nm=$(( nm + 1 )) fi if [ "$DOBLL_WAV" = YES ]; then - echo "$nm $USHwave/wave_tar.sh $WAV_MOD_TAG bull $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 " >> cmdtarfile +# echo "$nm $USHwave/wave_tar.sh $WAV_MOD_TAG bull $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 " >> cmdtarfile + echo "$USHwave/wave_tar.sh $WAV_MOD_TAG bull $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 " >> cmdtarfile nm=$(( nm + 1 )) - echo "$nm $USHwave/wave_tar.sh $WAV_MOD_TAG cbull $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 " >> cmdtarfile +# echo "$nm $USHwave/wave_tar.sh $WAV_MOD_TAG cbull $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 " >> cmdtarfile + echo "$USHwave/wave_tar.sh $WAV_MOD_TAG cbull $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 " >> cmdtarfile nm=$(( nm + 1 )) fi fi @@ -670,7 +793,7 @@ if [ "$wavenproc" -gt '1' ] then if [ ${CFP_MP:-"NO"} = "YES" ]; then - ${wavempexec} -n ${wavenproc} ${wave_mpmd} cmdtarfile + ${wavempexec} ${wave_mpmd} ${DATA}/cmdtarfile else ${wavempexec} ${wavenproc} ${wave_mpmd} cmdtarfile fi From 8086f493d657878ce8bbc69bcde059ad009299a6 Mon Sep 17 00:00:00 2001 From: AliS-Noaa Date: Fri, 3 Jan 2025 05:15:54 +0000 Subject: [PATCH 02/22] cleaned the outp script, issue #3157 --- scripts/exgfs_wave_post_pnt.sh | 291 +++------------------------------ 1 file changed, 23 insertions(+), 268 deletions(-) diff --git a/scripts/exgfs_wave_post_pnt.sh b/scripts/exgfs_wave_post_pnt.sh index 27d1375373..0eec91d5e0 100755 --- a/scripts/exgfs_wave_post_pnt.sh +++ b/scripts/exgfs_wave_post_pnt.sh @@ -345,7 +345,6 @@ echo ' Making command file for wave post points ' [[ "$LOUD" = YES ]] && set -x - if true; then grep -F -f ibp_tags buoy_lst.loc | awk '{ print $1 }' > buoys grep -F -f buoys buoy_log.ww3 | awk '{ print $1 }' > points rm buoys @@ -403,7 +402,7 @@ EOF fi - rm points + #rm points N=$(( ($FHMAX_WAV_PNT - $FHMIN_WAV) + 1 )) tstart="`echo $ymdh | cut -c1-8` `echo $ymdh | cut -c9-10`0000" truntime="`echo $CDATE | cut -c1-8` `echo $CDATE | cut -c9-10`0000" @@ -438,279 +437,35 @@ EOF fhr=$fhrp # no gridded output, loop with out_pnt stride done - $EXECwave/ww3_outp ${WAV_MOD_TAG} 1> ww3_outp.log 2>&1 + if [ "$DOSPC_WAV" = "YES" ] && [ "$DOBLL_WAV" = "YES" ]; then + cat << EOF > ww3_outp_auto.inp +$ WAVEWATCH III Point output post-processing +$ ------------------------------------------ + TIME DT COUNT +$ +EOF + cat points >> ww3_outp_auto.inp + cat << EOF >> ww3_outp_auto.inp + -1 +$ + 4 + 4 1 REFT 'UTC' +$ +$ End of input file +EOF - else - rm -f cmdfile - touch cmdfile - chmod 744 cmdfile - -# 1.a.2 Loop over forecast time to generate post files - fhr=$FHMIN_WAV - while [ $fhr -le $FHMAX_WAV_PNT ]; do - - echo " Creating the wave point scripts at : `date`" - ymdh=`$NDATE $fhr $CDATE` - YMD=$(echo $ymdh | cut -c1-8) - HMS="$(echo $ymdh | cut -c9-10)0000" - YMDHMS=${YMD}${HMS} - FH3=$(printf %03i $fhr) - - rm -f tmpcmdfile.${FH3} - touch tmpcmdfile.${FH3} - mkdir output_$YMDHMS - cd output_$YMDHMS - -# Create instances of directories for spec and gridded output - export SPECDATA=${DATA}/output_$YMDHMS - export BULLDATA=${DATA}/output_$YMDHMS - cp $DATA/mod_def.${waveuoutpGRD} mod_def.${waveuoutpGRD} - - pfile=$COMIN/rundata/${WAV_MOD_TAG}.out_pnt.${waveuoutpGRD}.${YMD}.${HMS} - if [ -f ${pfile} ] - then - ln -fs ${pfile} ./out_pnt.${waveuoutpGRD} - else - echo " FATAL ERROR : NO RAW POINT OUTPUT FILE out_pnt.$waveuoutpGRD.${YMD}.${HMS} " - echo ' ' - [[ "$LOUD" = YES ]] && set -x - err=7; export err;${errchk} - exit $err - fi - - cd $DATA - - if [ "$DOSPC_WAV" = 'YES' ] - then - export dtspec=3600. - for buoy in $buoys - do - echo "$USHwave/wave_outp_spec.sh $buoy $ymdh spec $SPECDATA > $SPECDATA/spec_$buoy.out 2>&1" >> tmpcmdfile.$FH3 - #echo "strace -o strace.${buoy}.${ymdh} $USHwave/wave_outp_spec.sh $buoy $ymdh spec $SPECDATA > $SPECDATA/spec_$buoy.out 2>&1" >> tmpcmdfile.$FH3 - done - fi - - if [ "$DOBLL_WAV" = 'YES' ] - then - export dtspec=3600. - for buoy in $buoys - do - #echo "$USHwave/wave_outp_spec.sh $buoy $ymdh bull $SPECDATA > $SPECDATA/bull_$buoy.out 2>&1" >> tmpcmdfile.$FH3 - #echo "LD_LIBRARY_PATH="/lib64:/usr/lib64:${LD_LIBRARY_PATH}" && $USHwave/wave_outp_spec.sh $buoy $ymdh bull $SPECDATA > $SPECDATA/bull_$buoy.out 2>&1" >> tmpcmdfile.$FH3 - #echo "/usr/bin/mkdir -p /tmp/output_$YMDHMS; $USHwave/wave_outp_spec.sh $buoy $ymdh bull /tmp/output_$YMDHMS > $SPECDATA/bull_$buoy.out 2>&1" >> tmpcmdfile.$FH3 - echo "/usr/bin/mkdir -p /tmp/output_$YMDHMS; LD_LIBRARY_PATH="/lib64:/usr/lib64:${LD_LIBRARY_PATH}" && $USHwave/wave_outp_spec.sh $buoy $ymdh bull /tmp/output_$YMDHMS > $SPECDATA/bull_$buoy.out 2>&1" >> tmpcmdfile.$FH3 - #echo "/usr/bin/mkdir -p /${SCRATCH}/output_$YMDHMS; LD_LIBRARY_PATH="/lib64:/usr/lib64:${LD_LIBRARY_PATH}" && $USHwave/wave_outp_spec.sh $buoy $ymdh bull /${SCRATCH}/output_$YMDHMS > $SPECDATA/bull_$buoy.out 2>&1" >> tmpcmdfile.$FH3 - #echo "/usr/bin/strace -o strace.${buoy}.${ymdh} $USHwave/wave_outp_spec.sh $buoy $ymdh bull $SPECDATA > $SPECDATA/bull_$buoy.out 2>&1" >> tmpcmdfile.$FH3 - done - fi - - for n in $(seq -w 1 10) - do - echo "export LD_LIBRARY_PATH=\"/lib64:/usr/lib64:${LD_LIBRARY_PATH}\"" > cmdfile.${FH3}.${n} - done - - split -n l/1/10 tmpcmdfile.$FH3 >> cmdfile.${FH3}.01 - split -n l/2/10 tmpcmdfile.$FH3 >> cmdfile.${FH3}.02 - split -n l/3/10 tmpcmdfile.$FH3 >> cmdfile.${FH3}.03 - split -n l/4/10 tmpcmdfile.$FH3 >> cmdfile.${FH3}.04 - split -n l/5/10 tmpcmdfile.$FH3 >> cmdfile.${FH3}.05 - split -n l/6/10 tmpcmdfile.$FH3 >> cmdfile.${FH3}.06 - split -n l/7/10 tmpcmdfile.$FH3 >> cmdfile.${FH3}.07 - split -n l/8/10 tmpcmdfile.$FH3 >> cmdfile.${FH3}.08 - split -n l/9/10 tmpcmdfile.$FH3 >> cmdfile.${FH3}.09 - split -n l/10/10 tmpcmdfile.$FH3 >> cmdfile.${FH3}.10 - - rm tmpcmdfile.$FH3 - chmod 744 cmdfile.${FH3}.01 cmdfile.${FH3}.02 cmdfile.${FH3}.03 cmdfile.${FH3}.04 - chmod 744 cmdfile.${FH3}.05 cmdfile.${FH3}.06 cmdfile.${FH3}.07 cmdfile.${FH3}.08 - chmod 744 cmdfile.${FH3}.09 cmdfile.${FH3}.10 - echo "$DATA/cmdfile.${FH3}.01" >> cmdfile - echo "$DATA/cmdfile.${FH3}.02" >> cmdfile - echo "$DATA/cmdfile.${FH3}.03" >> cmdfile - echo "$DATA/cmdfile.${FH3}.04" >> cmdfile - echo "$DATA/cmdfile.${FH3}.05" >> cmdfile - echo "$DATA/cmdfile.${FH3}.06" >> cmdfile - echo "$DATA/cmdfile.${FH3}.07" >> cmdfile - echo "$DATA/cmdfile.${FH3}.08" >> cmdfile - echo "$DATA/cmdfile.${FH3}.09" >> cmdfile - echo "$DATA/cmdfile.${FH3}.10" >> cmdfile - - - FHINCP=$(( DTPNT_WAV / 3600 )) - fhrp=$((fhr+FHINCP)) - fhr=$fhrp # no gridded output, loop with out_pnt stride - - done - - - if [ ${CFP_MP:-"NO"} = "YES" ]; then - nfile=0 - ifile=0 - iline=1 - ifirst='yes' - nlines=$( wc -l cmdfile | awk '{print $1}' ) - while [ $iline -le $nlines ]; do - line=$( sed -n ''$iline'p' cmdfile ) - if [ -z "$line" ]; then - break - else - if [ "$ifirst" = 'yes' ]; then - echo "#!/bin/sh" > cmdmfile.$nfile -# echo "$nfile cmdmfile.$nfile" >> cmdmprog - echo "$DATA/cmdmfile.$nfile" >> cmdmprog - chmod 744 cmdmfile.$nfile - fi - echo $line >> cmdmfile.$nfile - nfile=$(( nfile + 1 )) - if [ $nfile -eq $NTASKS ]; then - nfile=0 - ifirst='no' - fi - iline=$(( iline + 1 )) - fi - done - fi - - wavenproc=`wc -l cmdfile | awk '{print $1}'` - wavenproc=`echo $((${wavenproc}<${NTASKS}?${wavenproc}:${NTASKS}))` - - set +x - echo ' ' - echo " Executing the wave point scripts at : `date`" - echo ' ------------------------------------' - echo ' ' - [[ "$LOUD" = YES ]] && set -x - - if [ "$wavenproc" -gt '1' ] - then - if [ ${CFP_MP:-"NO"} = "YES" ]; then - echo "exgfs_wave_post_pnt: Running cmdmprog using ${wavempexec} on ${wavenproc}" - ${wavempexec} ${wave_mpmd} ${DATA}/cmdmprog - else - echo "exgfs_wave_post_pnt: Running cmdfile using ${wavempexec} on ${wavenproc}. ${NTASKS}" - ${wavempexec} ${wavenproc} ${wave_mpmd} cmdfile - fi - exit=$? - else - chmod 744 cmdfile - echo "exgfs_wave_post_pnt: Running cmdfile serially" - ./cmdfile - exit=$? - fi - - if [ "$exit" != '0' ] - then - set +x - echo ' ' - echo '*************************************' - echo '*** FATAL ERROR: CMDFILE FAILED ***' - echo '*************************************' - echo ' See Details Below ' - echo ' ' - [[ "$LOUD" = YES ]] && set -x - err=8; export err;${errchk} - exit $err - fi - -# 2.b Loop over each buoy to cat the final buoy file for all fhr - - cd $DATA - - echo "Before create cmdfile for cat bouy : `date`" - rm -f cmdfile.bouy - touch cmdfile.bouy - chmod 744 cmdfile.bouy - CATOUTDIR=${DATA}/pnt_cat_out - mkdir -p ${CATOUTDIR} - - if [ "$DOSPC_WAV" = 'YES' ] - then - for buoy in $buoys - do - echo "$USHwave/wave_outp_cat.sh $buoy $FHMAX_WAV_PNT spec > ${CATOUTDIR}/spec_cat_$buoy.out 2>&1" >> cmdfile.bouy - done - fi - - if [ "$DOBLL_WAV" = 'YES' ] - then - for buoy in $buoys - do - #echo "$USHwave/wave_outp_cat.sh $buoy $FHMAX_WAV_PNT bull > ${CATOUTDIR}/bull_cat_$buoy.out 2>&1" >> cmdfile.bouy - echo "LD_LIBRARY_PATH="/lib64:/usr/lib64:${LD_LIBRARY_PATH}" && $USHwave/wave_outp_cat.sh $buoy $FHMAX_WAV_PNT bull > ${CATOUTDIR}/bull_cat_$buoy.out 2>&1" >> cmdfile.bouy - #echo "LD_LIBRARY_PATH="/lib64:/usr/lib64:${LD_LIBRARY_PATH}" && /usr/bin/strace -o strace.${buoy}.${ymdh}.cat $USHwave/wave_outp_cat.sh $buoy $FHMAX_WAV_PNT bull > ${CATOUTDIR}/bull_cat_$buoy.out 2>&1" >> cmdfile.bouy - done - fi - - if [ ${CFP_MP:-"NO"} = "YES" ]; then - nfile=0 - ifile=0 - iline=1 - ifirst='yes' - nlines=$( wc -l cmdfile.bouy | awk '{print $1}' ) - while [ $iline -le $nlines ]; do - line=$( sed -n ''$iline'p' cmdfile.bouy ) - if [ -z "$line" ]; then - break - else - if [ "$ifirst" = 'yes' ]; then - echo "#!/bin/sh" > cmdfile.bouy.$nfile -# echo "$nfile cmdfile.bouy.$nfile" >> cmdmprogbouy - echo "$DATA/cmdfile.bouy.$nfile" >> cmdmprogbouy - chmod 744 cmdfile.bouy.$nfile - fi - echo $line >> cmdfile.bouy.$nfile - nfile=$(( nfile + 1 )) - if [ $nfile -eq $NTASKS ]; then - nfile=0 - ifirst='no' - fi - iline=$(( iline + 1 )) - fi - done - fi - - wavenproc=`wc -l cmdfile.bouy | awk '{print $1}'` - wavenproc=`echo $((${wavenproc}<${NTASKS}?${wavenproc}:${NTASKS}))` + rm points + sed -e "s/TIME/$tstart/g" \ + -e "s/DT/$dtspec/g" \ + -e "s/COUNT/$N/g" \ + -e "s/REFT/$truntime/g" ww3_outp_auto.inp > ww3_outp.inp - set +x - echo ' ' - echo " Executing the boundary point cat script at : `date`" - echo ' ------------------------------------' - echo ' ' - [[ "$LOUD" = YES ]] && set -x + $EXECwave/ww3_outp ${WAV_MOD_TAG} 1>> ww3_outp.log 2>&1 - if [ "$wavenproc" -gt '1' ] - then - if [ ${CFP_MP:-"NO"} = "YES" ]; then - ${wavempexec} ${wave_mpmd} ${DATA}/cmdmprogbouy - else - ${wavempexec} ${wavenproc} ${wave_mpmd} cmdfile.bouy - fi - exit=$? - else - chmod 744 ${fcmdnow} - ./${fcmdnow} - #./cmdfile.bouy - exit=$? - fi - if [ "$exit" != '0' ] - then - set +x - echo ' ' - echo '*************************************' - echo '*** FATAL ERROR: CMDFILE FAILED ***' - echo '*************************************' - echo ' See Details Below ' - echo ' ' - [[ "$LOUD" = YES ]] && set -x - err=9; export err;${errchk} - exit $err fi - fi # dkokron New versus Old - # --------------------------------------------------------------------------- # # 3. Compress point output data into tar files From 84fc8f3ed46545845e41331270300556123ed568 Mon Sep 17 00:00:00 2001 From: AliS-Noaa Date: Tue, 7 Jan 2025 20:47:09 +0000 Subject: [PATCH 03/22] changing the point output script and cleaning it and changing the resource allocatio, issue #3157 --- .../gfs/wave/post/jgfs_wave_post_bndpnt.ecf | 4 +- .../wave/post/jgfs_wave_post_bndpntbll.ecf | 4 +- .../gfs/wave/post/jgfs_wave_postpnt.ecf | 4 +- parm/config/config.resources.emc.dyn | 18 +- parm/config/config.resources.nco.static | 18 +- scripts/exgfs_wave_post_pnt.sh | 219 ++++++------------ 6 files changed, 101 insertions(+), 166 deletions(-) diff --git a/ecf/scripts/gfs/wave/post/jgfs_wave_post_bndpnt.ecf b/ecf/scripts/gfs/wave/post/jgfs_wave_post_bndpnt.ecf index 6db7a431bc..02ecc35868 100755 --- a/ecf/scripts/gfs/wave/post/jgfs_wave_post_bndpnt.ecf +++ b/ecf/scripts/gfs/wave/post/jgfs_wave_post_bndpnt.ecf @@ -3,8 +3,8 @@ #PBS -j oe #PBS -q %QUEUE% #PBS -A %PROJ%-%PROJENVIR% -#PBS -l walltime=01:00:00 -#PBS -l select=3:ncpus=80:ompthreads=1 +#PBS -l walltime=00:30:00 +#PBS -l select=1:ncpus=1:ompthreads=1 #PBS -l place=vscatter:exclhost #PBS -l debug=true diff --git a/ecf/scripts/gfs/wave/post/jgfs_wave_post_bndpntbll.ecf b/ecf/scripts/gfs/wave/post/jgfs_wave_post_bndpntbll.ecf index d285f31385..e869872b6f 100755 --- a/ecf/scripts/gfs/wave/post/jgfs_wave_post_bndpntbll.ecf +++ b/ecf/scripts/gfs/wave/post/jgfs_wave_post_bndpntbll.ecf @@ -3,8 +3,8 @@ #PBS -j oe #PBS -q %QUEUE% #PBS -A %PROJ%-%PROJENVIR% -#PBS -l walltime=01:00:00 -#PBS -l select=4:ncpus=112:ompthreads=1 +#PBS -l walltime=00:10:00 +#PBS -l select=1:ncpus=2:ompthreads=1 #PBS -l place=vscatter:exclhost #PBS -l debug=true diff --git a/ecf/scripts/gfs/wave/post/jgfs_wave_postpnt.ecf b/ecf/scripts/gfs/wave/post/jgfs_wave_postpnt.ecf index c09dbb110f..1b680fe514 100755 --- a/ecf/scripts/gfs/wave/post/jgfs_wave_postpnt.ecf +++ b/ecf/scripts/gfs/wave/post/jgfs_wave_postpnt.ecf @@ -3,8 +3,8 @@ #PBS -j oe #PBS -q %QUEUE% #PBS -A %PROJ%-%PROJENVIR% -#PBS -l walltime=01:30:00 -#PBS -l select=4:ncpus=50:ompthreads=1 +#PBS -l walltime=00:35:00 +#PBS -l select=1:ncpus=3:ompthreads=1 #PBS -l place=vscatter:exclhost #PBS -l debug=true diff --git a/parm/config/config.resources.emc.dyn b/parm/config/config.resources.emc.dyn index 8746bb4dc8..6a6e657390 100755 --- a/parm/config/config.resources.emc.dyn +++ b/parm/config/config.resources.emc.dyn @@ -82,26 +82,26 @@ elif [ $step = "wavepostsbs" ]; then elif [ $step = "wavepostbndpnt" ]; then - export wtime_wavepostbndpnt="01:00:00" - export npe_wavepostbndpnt=240 + export wtime_wavepostbndpnt="00:30:00" + export npe_wavepostbndpnt=1 export nth_wavepostbndpnt=1 - export npe_node_wavepostbndpnt=80 + export npe_node_wavepostbndpnt=1 export NTASKS=$npe_wavepostbndpnt elif [ $step = "wavepostbndpntbll" ]; then - export wtime_wavepostbndpntbll="01:00:00" - export npe_wavepostbndpntbll=448 + export wtime_wavepostbndpntbll="00:10:00" + export npe_wavepostbndpntbll=2 export nth_wavepostbndpntbll=1 - export npe_node_wavepostbndpntbll=112 + export npe_node_wavepostbndpntbll=2 export NTASKS=$npe_wavepostbndpntbll elif [ $step = "wavepostpnt" ]; then - export wtime_wavepostpnt="01:30:00" - export npe_wavepostpnt=200 + export wtime_wavepostpnt="00:35:00" + export npe_wavepostpnt=3 export nth_wavepostpnt=1 - export npe_node_wavepostpnt=50 + export npe_node_wavepostpnt=3 export NTASKS=$npe_wavepostpnt elif [ $step = "wavegempak" ]; then diff --git a/parm/config/config.resources.nco.static b/parm/config/config.resources.nco.static index e1e65357cb..f50faa1772 100755 --- a/parm/config/config.resources.nco.static +++ b/parm/config/config.resources.nco.static @@ -66,26 +66,26 @@ elif [ $step = "wavepostsbs" ]; then elif [ $step = "wavepostbndpnt" ]; then - export wtime_wavepostbndpnt="01:00:00" - export npe_wavepostbndpnt=240 + export wtime_wavepostbndpnt="00:30:00" + export npe_wavepostbndpnt=1 export nth_wavepostbndpnt=1 - export npe_node_wavepostbndpnt=80 + export npe_node_wavepostbndpnt=1 export NTASKS=$npe_wavepostbndpnt elif [ $step = "wavepostbndpntbll" ]; then - export wtime_wavepostbndpntbll="01:00:00" - export npe_wavepostbndpntbll=448 + export wtime_wavepostbndpntbll="00:10:00" + export npe_wavepostbndpntbll=2 export nth_wavepostbndpntbll=1 - export npe_node_wavepostbndpntbll=112 + export npe_node_wavepostbndpntbll=2 export NTASKS=$npe_wavepostbndpntbll elif [ $step = "wavepostpnt" ]; then - export wtime_wavepostpnt="01:30:00" - export npe_wavepostpnt=200 + export wtime_wavepostpnt="00:35:00" + export npe_wavepostpnt=3 export nth_wavepostpnt=1 - export npe_node_wavepostpnt=50 + export npe_node_wavepostpnt=3 export NTASKS=$npe_wavepostpnt elif [ $step = "wavegempak" ]; then diff --git a/scripts/exgfs_wave_post_pnt.sh b/scripts/exgfs_wave_post_pnt.sh index 0eec91d5e0..c64f310ca0 100755 --- a/scripts/exgfs_wave_post_pnt.sh +++ b/scripts/exgfs_wave_post_pnt.sh @@ -68,25 +68,19 @@ exit $err fi -# 0.c Defining model grids - - waveuoutpGRD=${waveuoutpGRD:?buoyNotSet} - -# 0.c.1 Define a temporary directory for storing ascii point output files -# and flush it - export STA_DIR=$DATA/station_ascii_files if [ -d $STA_DIR ] - then + then rm -rf ${STA_DIR} fi mkdir -p ${STA_DIR} mkdir -p ${STA_DIR}/spec mkdir -p ${STA_DIR}/bull mkdir -p ${STA_DIR}/cbull - mkdir -p ${STA_DIR}/specfhr - mkdir -p ${STA_DIR}/bullfhr - mkdir -p ${STA_DIR}/cbullfhr + +# 0.b Defining model grids + + waveuoutpGRD=${waveuoutpGRD:?buoyNotSet} set +x echo ' ' @@ -109,12 +103,6 @@ # 1.a Model definition files and output files (set up using poe) -# 1.a.1 Set up the parallel command tasks - - rm -f cmdfile - touch cmdfile - chmod 744 cmdfile - [[ "$LOUD" = YES ]] && set -x # Copy model definition files @@ -151,7 +139,7 @@ fi done -# 1.c Output locations file +# 1.b Output locations file rm -f buoy.loc @@ -186,7 +174,7 @@ DOBLL_WAV='NO' fi -# 1.d Input template files +# 1.c Input template files if [ -f $FIXwave/ww3_outp_spec.inp.tmpl ] then @@ -233,7 +221,7 @@ DOBLL_WAV='NO' fi -# 1.e Getting buoy information for points +# 1.d Getting buoy information for points if [ "$DOSPC_WAV" = 'YES' ] || [ "$DOBLL_WAV" = 'YES' ] then @@ -270,7 +258,6 @@ $EXECwave/ww3_outp ${WAV_MOD_TAG} > buoy_lst.loc 2>&1 export err=$?;err_chk - if [ "$err" != '0' ] && [ ! -f buoy_log.ww3 ] then pgm=wave_post @@ -319,10 +306,9 @@ DOSPC_WAV='NO' DOBLL_WAV='NO' fi - fi -# 1.f Data summary +# 1.e Data summary set +x echo ' ' @@ -339,133 +325,88 @@ # --------------------------------------------------------------------------- # # 2. Make files for processing boundary points # -# 2.a Command file set-up +# 2.a creating ww3_outp.inp for each jobs set +x - echo ' Making command file for wave post points ' + echo ' Making input file for wave post point ' [[ "$LOUD" = YES ]] && set -x - grep -F -f ibp_tags buoy_lst.loc | awk '{ print $1 }' > buoys - grep -F -f buoys buoy_log.ww3 | awk '{ print $1 }' > points - rm buoys - - if [ "$DOBLL_WAV" = "YES" ] && [ "$DOSPC_WAV" = "NO" ]; then - cat << EOF > ww3_outp_auto.inp -$ WAVEWATCH III Point output post-processing -$ ------------------------------------------ - TIME DT COUNT -$ -EOF - cat points >> ww3_outp_auto.inp - cat << EOF >> ww3_outp_auto.inp - -1 -$ - 4 - 4 1 REFT 'UTC' -$ -$ End of input file -EOF - - elif [ "$DOSPC_WAV" = "YES" ] && [ "$DOBLL_WAV" = "NO" ]; then - cat << EOF > ww3_outp_auto.inp -$ WAVEWATCH III Point output post-processing -$ ------------------------------------------ - TIME DT COUNT -$ -EOF - cat points >> ww3_outp_auto.inp - cat << EOF >> ww3_outp_auto.inp - -1 -$ - 1 - 3 0. 0. 51 FORMAT -$ -$ End of input file -EOF - - elif [ "$DOSPC_WAV" = "YES" ] && [ "$DOBLL_WAV" = "YES" ]; then - cat << EOF > ww3_outp_auto.inp -$ WAVEWATCH III Point output post-processing -$ ------------------------------------------ - TIME DT COUNT -$ -EOF - cat points >> ww3_outp_auto.inp - cat << EOF >> ww3_outp_auto.inp - -1 -$ - 1 - 3 0. 0. 51 FORMAT -$ -$ End of input file -EOF - + grep -F -f ibp_tags buoy_lst.loc | awk '{ print $1 }' > buoys + grep -F -f buoys buoy_log.ww3 | awk '{ print $1 }' > points + points=$(cat points | awk '{print $0 "\\n"}' | tr -d '\n') + rm buoys + + ymdh=`$NDATE -${WAVHINDH} $CDATE` + tstart="`echo $ymdh | cut -c1-8` `echo $ymdh | cut -c9-10`0000" + dtspec=3600. # default time step (not used here) + N=$(( ($FHMAX_WAV_PNT - $FHMIN_WAV) + 1 )) + truntime="`echo $CDATE | cut -c1-8` `echo $CDATE | cut -c9-10`0000" + + fhr=$FHMIN_WAV + while [ $fhr -le $FHMAX_WAV_PNT ]; do + ymdh=`$NDATE $fhr $CDATE` + YMD=$(echo $ymdh | cut -c1-8) + HMS="$(echo $ymdh | cut -c9-10)0000" + + pfile=$COMIN/rundata/${WAV_MOD_TAG}.out_pnt.${waveuoutpGRD}.${YMD}.${HMS} + if [ -f ${pfile} ] + then + ln -fs ${pfile} + else + echo " FATAL ERROR : NO RAW POINT OUTPUT FILE out_pnt.$waveuoutpGRD.${YMD}.${HMS} " + echo ' ' + [[ "$LOUD" = YES ]] && set -x + err=7; export err;${errchk} + exit $err fi + FHINCP=$(( DTPNT_WAV / 3600 )) + fhrp=$((fhr+FHINCP)) + fhr=$fhrp # no gridded output, loop with out_pnt stride + done - #rm points - N=$(( ($FHMAX_WAV_PNT - $FHMIN_WAV) + 1 )) - tstart="`echo $ymdh | cut -c1-8` `echo $ymdh | cut -c9-10`0000" - truntime="`echo $CDATE | cut -c1-8` `echo $CDATE | cut -c9-10`0000" + if [ "$DOSPC_WAV" = 'YES' ] && [ "$DOBLL_WAV" = "NO" ]; then sed -e "s/TIME/$tstart/g" \ -e "s/DT/$dtspec/g" \ - -e "s/COUNT/$N/g" \ - -e "s/REFT/$truntime/g" ww3_outp_auto.inp > ww3_outp.inp - - cp $DATA/mod_def.${waveuoutpGRD} mod_def.${waveuoutpGRD} - - fhr=$FHMIN_WAV - while [ $fhr -le $FHMAX_WAV_PNT ]; do - - #echo " Creating the wave point scripts at : `date`" - ymdh=`$NDATE $fhr $CDATE` - YMD=$(echo $ymdh | cut -c1-8) - HMS="$(echo $ymdh | cut -c9-10)0000" - - pfile=$COMIN/rundata/${WAV_MOD_TAG}.out_pnt.${waveuoutpGRD}.${YMD}.${HMS} - if [ -f ${pfile} ] - then - ln -fs ${pfile} - else - echo " FATAL ERROR : NO RAW POINT OUTPUT FILE out_pnt.$waveuoutpGRD.${YMD}.${HMS} " - echo ' ' - [[ "$LOUD" = YES ]] && set -x - err=7; export err;${errchk} - exit $err - fi - FHINCP=$(( DTPNT_WAV / 3600 )) - fhrp=$((fhr+FHINCP)) - fhr=$fhrp # no gridded output, loop with out_pnt stride - done - - $EXECwave/ww3_outp ${WAV_MOD_TAG} 1> ww3_outp.log 2>&1 - - if [ "$DOSPC_WAV" = "YES" ] && [ "$DOBLL_WAV" = "YES" ]; then - cat << EOF > ww3_outp_auto.inp -$ WAVEWATCH III Point output post-processing -$ ------------------------------------------ - TIME DT COUNT -$ -EOF - cat points >> ww3_outp_auto.inp - cat << EOF >> ww3_outp_auto.inp - -1 -$ - 4 - 4 1 REFT 'UTC' -$ -$ End of input file -EOF - - rm points + -e "s/999/$N/g" \ + -e "s|POINT|$points|g" \ + -e "s/ITYPE/1/g" \ + -e "s/FORMAT/F/g" \ + ww3_outp_spec.inp.tmpl > ww3_outp.inp + + $EXECwave/ww3_outp ${WAV_MOD_TAG} 1> ww3_outp_spec.log 2>&1 + + elif [ "$DOSPC_WAV" = 'NO' ] && [ "$DOBLL_WAV" = "YES" ]; then sed -e "s/TIME/$tstart/g" \ -e "s/DT/$dtspec/g" \ - -e "s/COUNT/$N/g" \ - -e "s/REFT/$truntime/g" ww3_outp_auto.inp > ww3_outp.inp + -e "s/999/$N/g" \ + -e "s|POINT|$points|g" \ + -e "s/REFT/$truntime/g" \ + ww3_outp_bull.inp.tmpl > ww3_outp.inp + + $EXECwave/ww3_outp ${WAV_MOD_TAG} 1> ww3_outp_bull.log 2>&1 - $EXECwave/ww3_outp ${WAV_MOD_TAG} 1>> ww3_outp.log 2>&1 + elif [[ "$DOSPC_WAV" == "YES" && "$DOBLL_WAV" == "YES" ]]; then + sed -e "s/TIME/$tstart/g" \ + -e "s/DT/$dtspec/g" \ + -e "s/999/$N/g" \ + -e "s|POINT|$points|g" \ + -e "s/ITYPE/1/g" \ + -e "s/FORMAT/F/g" \ + ww3_outp_spec.inp.tmpl > ww3_outp.inp + + $EXECwave/ww3_outp ${WAV_MOD_TAG} 1> ww3_outp_spec.log 2>&1 + sed -e "s/TIME/$tstart/g" \ + -e "s/DT/$dtspec/g" \ + -e "s/999/$N/g" \ + -e "s|POINT|$points|g" \ + -e "s/REFT/$truntime/g" \ + ww3_outp_bull.inp.tmpl > ww3_outp.inp + + $EXECwave/ww3_outp ${WAV_MOD_TAG} 1> ww3_outp_bull.log 2>&1 fi + # --------------------------------------------------------------------------- # # 3. Compress point output data into tar files @@ -488,29 +429,23 @@ EOF if [ ${CFP_MP:-"NO"} = "YES" ]; then if [ "$DOBNDPNT_WAV" = YES ]; then if [ "$DOSPC_WAV" = YES ]; then -# echo "$nm $USHwave/wave_tar.sh $WAV_MOD_TAG ibp $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 " >> cmdtarfile echo "$USHwave/wave_tar.sh $WAV_MOD_TAG ibp $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 " >> cmdtarfile nm=$(( nm + 1 )) fi if [ "$DOBLL_WAV" = YES ]; then -# echo "$nm $USHwave/wave_tar.sh $WAV_MOD_TAG ibpbull $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 " >> cmdtarfile echo "$USHwave/wave_tar.sh $WAV_MOD_TAG ibpbull $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 " >> cmdtarfile nm=$(( nm + 1 )) -# echo "$nm $USHwave/wave_tar.sh $WAV_MOD_TAG ibpcbull $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 " >> cmdtarfile echo "$USHwave/wave_tar.sh $WAV_MOD_TAG ibpcbull $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 " >> cmdtarfile nm=$(( nm + 1 )) fi else if [ "$DOSPC_WAV" = YES ]; then -# echo "$nm $USHwave/wave_tar.sh $WAV_MOD_TAG spec $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 " >> cmdtarfile echo "$USHwave/wave_tar.sh $WAV_MOD_TAG spec $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 " >> cmdtarfile nm=$(( nm + 1 )) fi if [ "$DOBLL_WAV" = YES ]; then -# echo "$nm $USHwave/wave_tar.sh $WAV_MOD_TAG bull $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 " >> cmdtarfile echo "$USHwave/wave_tar.sh $WAV_MOD_TAG bull $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 " >> cmdtarfile nm=$(( nm + 1 )) -# echo "$nm $USHwave/wave_tar.sh $WAV_MOD_TAG cbull $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 " >> cmdtarfile echo "$USHwave/wave_tar.sh $WAV_MOD_TAG cbull $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 " >> cmdtarfile nm=$(( nm + 1 )) fi From d2b6b70ba1db948b736e40df3f2cc2948afe6fd1 Mon Sep 17 00:00:00 2001 From: AliS-Noaa Date: Wed, 8 Jan 2025 17:18:59 +0000 Subject: [PATCH 04/22] concising the exgfs_wave_post_pnt.sh and config files, issue #3157 --- parm/config/config.resources.emc.dyn | 6 ++--- parm/config/config.resources.nco.static | 6 ++--- scripts/exgfs_wave_post_pnt.sh | 29 +++++-------------------- 3 files changed, 11 insertions(+), 30 deletions(-) diff --git a/parm/config/config.resources.emc.dyn b/parm/config/config.resources.emc.dyn index 6a6e657390..5236159121 100755 --- a/parm/config/config.resources.emc.dyn +++ b/parm/config/config.resources.emc.dyn @@ -85,7 +85,7 @@ elif [ $step = "wavepostbndpnt" ]; then export wtime_wavepostbndpnt="00:30:00" export npe_wavepostbndpnt=1 export nth_wavepostbndpnt=1 - export npe_node_wavepostbndpnt=1 + export npe_node_wavepostbndpnt=${npe_wavepostbndpnt} export NTASKS=$npe_wavepostbndpnt elif [ $step = "wavepostbndpntbll" ]; then @@ -93,7 +93,7 @@ elif [ $step = "wavepostbndpntbll" ]; then export wtime_wavepostbndpntbll="00:10:00" export npe_wavepostbndpntbll=2 export nth_wavepostbndpntbll=1 - export npe_node_wavepostbndpntbll=2 + export npe_node_wavepostbndpntbll=${npe_wavepostbndpntbll} export NTASKS=$npe_wavepostbndpntbll elif [ $step = "wavepostpnt" ]; then @@ -101,7 +101,7 @@ elif [ $step = "wavepostpnt" ]; then export wtime_wavepostpnt="00:35:00" export npe_wavepostpnt=3 export nth_wavepostpnt=1 - export npe_node_wavepostpnt=3 + export npe_node_wavepostpnt=${npe_wavepostpnt} export NTASKS=$npe_wavepostpnt elif [ $step = "wavegempak" ]; then diff --git a/parm/config/config.resources.nco.static b/parm/config/config.resources.nco.static index f50faa1772..4e22917b25 100755 --- a/parm/config/config.resources.nco.static +++ b/parm/config/config.resources.nco.static @@ -69,7 +69,7 @@ elif [ $step = "wavepostbndpnt" ]; then export wtime_wavepostbndpnt="00:30:00" export npe_wavepostbndpnt=1 export nth_wavepostbndpnt=1 - export npe_node_wavepostbndpnt=1 + export npe_node_wavepostbndpnt=${npe_wavepostbndpnt} export NTASKS=$npe_wavepostbndpnt elif [ $step = "wavepostbndpntbll" ]; then @@ -77,7 +77,7 @@ elif [ $step = "wavepostbndpntbll" ]; then export wtime_wavepostbndpntbll="00:10:00" export npe_wavepostbndpntbll=2 export nth_wavepostbndpntbll=1 - export npe_node_wavepostbndpntbll=2 + export npe_node_wavepostbndpntbll=${npe_wavepostbndpntbll} export NTASKS=$npe_wavepostbndpntbll elif [ $step = "wavepostpnt" ]; then @@ -85,7 +85,7 @@ elif [ $step = "wavepostpnt" ]; then export wtime_wavepostpnt="00:35:00" export npe_wavepostpnt=3 export nth_wavepostpnt=1 - export npe_node_wavepostpnt=3 + export npe_node_wavepostpnt=${npe_wavepostpnt} export NTASKS=$npe_wavepostpnt elif [ $step = "wavegempak" ]; then diff --git a/scripts/exgfs_wave_post_pnt.sh b/scripts/exgfs_wave_post_pnt.sh index c64f310ca0..7875816828 100755 --- a/scripts/exgfs_wave_post_pnt.sh +++ b/scripts/exgfs_wave_post_pnt.sh @@ -364,7 +364,7 @@ fhr=$fhrp # no gridded output, loop with out_pnt stride done - if [ "$DOSPC_WAV" = 'YES' ] && [ "$DOBLL_WAV" = "NO" ]; then + if [ "$DOSPC_WAV" = 'YES' ]; then sed -e "s/TIME/$tstart/g" \ -e "s/DT/$dtspec/g" \ -e "s/999/$N/g" \ @@ -372,10 +372,12 @@ -e "s/ITYPE/1/g" \ -e "s/FORMAT/F/g" \ ww3_outp_spec.inp.tmpl > ww3_outp.inp - + $EXECwave/ww3_outp ${WAV_MOD_TAG} 1> ww3_outp_spec.log 2>&1 - elif [ "$DOSPC_WAV" = 'NO' ] && [ "$DOBLL_WAV" = "YES" ]; then + fi + + if [ "$DOBLL_WAV" = "YES" ]; then sed -e "s/TIME/$tstart/g" \ -e "s/DT/$dtspec/g" \ -e "s/999/$N/g" \ @@ -384,27 +386,6 @@ ww3_outp_bull.inp.tmpl > ww3_outp.inp $EXECwave/ww3_outp ${WAV_MOD_TAG} 1> ww3_outp_bull.log 2>&1 - - elif [[ "$DOSPC_WAV" == "YES" && "$DOBLL_WAV" == "YES" ]]; then - sed -e "s/TIME/$tstart/g" \ - -e "s/DT/$dtspec/g" \ - -e "s/999/$N/g" \ - -e "s|POINT|$points|g" \ - -e "s/ITYPE/1/g" \ - -e "s/FORMAT/F/g" \ - ww3_outp_spec.inp.tmpl > ww3_outp.inp - - $EXECwave/ww3_outp ${WAV_MOD_TAG} 1> ww3_outp_spec.log 2>&1 - - sed -e "s/TIME/$tstart/g" \ - -e "s/DT/$dtspec/g" \ - -e "s/999/$N/g" \ - -e "s|POINT|$points|g" \ - -e "s/REFT/$truntime/g" \ - ww3_outp_bull.inp.tmpl > ww3_outp.inp - - $EXECwave/ww3_outp ${WAV_MOD_TAG} 1> ww3_outp_bull.log 2>&1 - fi # --------------------------------------------------------------------------- # From d122080e1e170398baf1c994456b38cda0956490 Mon Sep 17 00:00:00 2001 From: AliS-Noaa Date: Tue, 21 Jan 2025 02:30:24 +0000 Subject: [PATCH 05/22] fixing tar script for the optimized point output, issue #3157 --- ush/wave_tar.sh | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/ush/wave_tar.sh b/ush/wave_tar.sh index b2054ab4d8..de16356da7 100755 --- a/ush/wave_tar.sh +++ b/ush/wave_tar.sh @@ -92,8 +92,6 @@ exit 2 fi - cd ${STA_DIR}/${filext} - # --------------------------------------------------------------------------- # # 2. Generate tar file (spectral files are compressed) @@ -109,7 +107,7 @@ while [ "$count" -lt "$countMAX" ] && [ "$tardone" = 'no' ] do - nf=`ls | awk '/'$ID.*.$filext'/ {a++} END {print a}'` + nf=`ls -1 $ID.*.$filext | awk 'END{print NR}'` nbm2=$(( $nb - 2 )) if [ $nf -ge $nbm2 ] then @@ -202,7 +200,6 @@ exit 4 fi - # if [ "$SENDDBN" = 'YES' -a $type != "ibp" ] if [ "$SENDDBN" = 'YES' ] then set +x From 8e2ad855898c670c1bfe8e65d3ee1a611a6218bd Mon Sep 17 00:00:00 2001 From: AliS-Noaa Date: Tue, 21 Jan 2025 15:56:32 +0000 Subject: [PATCH 06/22] retreiving the file counting, issue #3157 --- ush/wave_tar.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/wave_tar.sh b/ush/wave_tar.sh index de16356da7..b53ab06704 100755 --- a/ush/wave_tar.sh +++ b/ush/wave_tar.sh @@ -107,7 +107,7 @@ while [ "$count" -lt "$countMAX" ] && [ "$tardone" = 'no' ] do - nf=`ls -1 $ID.*.$filext | awk 'END{print NR}'` + nf=$(ls | awk '/'$ID.*.$filext'/ {a++} END {print a}') nbm2=$(( $nb - 2 )) if [ $nf -ge $nbm2 ] then From 1a9b7f91549c82a348fcff57d021c7dd7c9560b6 Mon Sep 17 00:00:00 2001 From: AliS-Noaa Date: Fri, 24 Jan 2025 21:19:06 +0000 Subject: [PATCH 07/22] Revising the checkout.sh --- sorc/checkout.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sorc/checkout.sh b/sorc/checkout.sh index cb4d80ffdb..b6e6960f7a 100755 --- a/sorc/checkout.sh +++ b/sorc/checkout.sh @@ -23,7 +23,8 @@ echo $topdir echo fv3gfs checkout ... if [[ ! -d fv3gfs.fd ]] ; then rm -f ${topdir}/checkout-fv3gfs.log - git clone --recursive --branch GFS.v16.3.1 https://github.com/ufs-community/ufs-weather-model.git fv3gfs.fd >> ${topdir}/checkout-fv3gfs.log 2>&1 + git clone --recursive --branch prodpoint https://github.com/AliS-Noaa/ufs-weather-model.git fv3gfs.fd >> ${topdir}/checkout-fv3gfs.log 2>&1 + #git clone --recursive --branch GFS.v16.3.1 https://github.com/ufs-community/ufs-weather-model.git fv3gfs.fd >> ${topdir}/checkout-fv3gfs.log 2>&1 cd ${topdir} else echo 'Skip. Directory fv3gfs.fd already exists.' From 57840534dc2e71f8702f8843cf57d4dffbf0cfe1 Mon Sep 17 00:00:00 2001 From: AliS-Noaa Date: Tue, 28 Jan 2025 15:32:40 +0000 Subject: [PATCH 08/22] fixing the links to inputs for wave post script --- scripts/exgfs_wave_post_pnt.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/exgfs_wave_post_pnt.sh b/scripts/exgfs_wave_post_pnt.sh index 7875816828..26c139b141 100755 --- a/scripts/exgfs_wave_post_pnt.sh +++ b/scripts/exgfs_wave_post_pnt.sh @@ -240,7 +240,7 @@ HMS="$(echo $CDATE | cut -c9-10)0000" if [ -f $COMIN/rundata/${WAV_MOD_TAG}.out_pnt.${waveuoutpGRD}.${YMD}.${HMS} ] then - ln -s $COMIN/rundata/${WAV_MOD_TAG}.out_pnt.${waveuoutpGRD}.${YMD}.${HMS} + ln -s $COMIN/rundata/${WAV_MOD_TAG}.out_pnt.${waveuoutpGRD}.${YMD}.${HMS} ./${WAV_MOD_TAG}.out_pnt.${waveuoutpGRD}.${YMD}.${HMS} else echo '*************************************************** ' echo " FATAL ERROR : NO RAW POINT OUTPUT FILE out_pnt.${waveuoutpGRD}.${YMD}.${HMS} " @@ -351,7 +351,7 @@ pfile=$COMIN/rundata/${WAV_MOD_TAG}.out_pnt.${waveuoutpGRD}.${YMD}.${HMS} if [ -f ${pfile} ] then - ln -fs ${pfile} + ln -fs ${pfile} ./${WAV_MOD_TAG}.out_pnt.${waveuoutpGRD}.${YMD}.${HMS} else echo " FATAL ERROR : NO RAW POINT OUTPUT FILE out_pnt.$waveuoutpGRD.${YMD}.${HMS} " echo ' ' From 14059bf1afac3fe746508252ccb25492f481a15f Mon Sep 17 00:00:00 2001 From: AliS-Noaa Date: Tue, 28 Jan 2025 19:42:36 +0000 Subject: [PATCH 09/22] Updatuing the Release_Notes.md for wave point output update --- docs/Release_Notes.md | 48 +++++++++++++++++++------------------------ 1 file changed, 21 insertions(+), 27 deletions(-) diff --git a/docs/Release_Notes.md b/docs/Release_Notes.md index 1a36c59864..fea9c61614 100644 --- a/docs/Release_Notes.md +++ b/docs/Release_Notes.md @@ -4,9 +4,7 @@ GFS V16.3.22 RELEASE NOTES PRELUDE ------- -The upstream OBSPROC package is updated to v1.3. Along with this are the following companion updates: -* workflow and UFS_UTILS package updates to use the new AFWA global snow file due to the hemispheric snow files being phased out -* updated GSI code and convinfo file for saildrone observations +The WW3 point output program (ww3_outp) is improved to process per-time-step point output to reduce the resources for the wave_post_pnt jobs. IMPLEMENTATION INSTRUCTIONS --------------------------- @@ -26,7 +24,7 @@ The checkout script extracts the following GFS components: | Component | Tag | POC | | --------- | ----------- | ----------------- | -| MODEL | GFS.v16.3.1 | Jun.Wang@noaa.gov | +| MODEL | GFS.v16.3.22 | Jun.Wang@noaa.gov | | GLDAS | gldas_gfsv16_release.v.2.1.0 | Helin.Wei@noaa.gov | | GSI | gfsda.v16.3.22 | Andrew.Collard@noaa.gov | | UFS_UTILS | ops-gfsv16.3.20 | George.Gayno@noaa.gov | @@ -51,85 +49,81 @@ cd ../ecf VERSION FILE CHANGES -------------------- -* `versions/run.ver` - change `version=v16.3.22`, `gfs_ver=v16.3.22`, and `obsproc_ver=v1.3` +* No changes from GFS v16.3.22 SORC CHANGES ------------ -* New UFS_UTILS tag - `emcsfc_snow2mdl` program and associated scripts are updated to process global AFWA snow data -* New GSI tag - `src/gsi/read_prepbufr.f90` code update for new saildrone subtype +* `ww3_outp` program and associated scripts are improved to process the per-time-step point outputs more efficiently. JOBS CHANGES ------------ -* `jobs/JGLOBAL_ATMOS_EMCSFC_SFC_PREP` - new AFWA filename +* No changes from GFS v16.3.22 PARM/CONFIG CHANGES ------------------- -* No changes from GFS v16.3.21 +* `config.resources.emc.dyn` and `config.resources.nco.static` are changed to reduce the resources accordingly. SCRIPT CHANGES -------------- -* No changes from GFS v16.3.21 +* `scripts/exgfs_wave_post_pnt.sh` is changed to create be compatible with the new `ww3_outp`. FIX CHANGES ----------- -* GSI `global_convinfo.txt` fix update for saildrone +* No changes from GFS v16.3.22 MODULE CHANGES -------------- -* No changes from GFS v16.3.21 +* No changes from GFS v16.3.22 CHANGES TO FILE AND FILE SIZES ------------------------------ -No longer ingest: -* `${RUN}.${cycle}.NPR.SNWN.SP.S1200.MESH16.grb` (`AFWA_NH_FILE`) -* `${RUN}.${cycle}.NPR.SNWS.SP.S1200.MESH16.grb` (`AFWA_SH_FILE`) +No longer uses the parallel command file `cmdfile` for the `scripts/exgfs_wave_post_pnt.sh`. -Now ingest: -* `${RUN}.${cycle}.snow.usaf.grib2` (`AFWA_GLOBAL_FILE`) ENVIRONMENT AND RESOURCE CHANGES -------------------------------- -* Increase ncpus from 50 to 60 for gfs_wave_postpnt job. - NCO provided change to avoid ncpu burst warnings. +Reduce the ncpu and wtime as follow: +* for jobs/JGLOBAL_WAVE_POST_BNDPNT; ncpu from 240 to 1; wtime from 1hr to 30min +* for jobs/JGLOBAL_WAVE_POST_BNDPNTBLL; ncpu from 448 to 2; wtime from 1hr to 10min +* for jobs/JGLOBAL_WAVE_POST_PNT; ncpu from 200 to 3; wtime from 1.5hr to 35min PRE-IMPLEMENTATION TESTING REQUIREMENTS --------------------------------------- * Which production jobs should be tested as part of this implementation? - * emcsfc_sfc_prep and analysis + * wave_post_pnt * Does this change require a 30-day evaluation? * No DISSEMINATION INFORMATION ------------------------- -* No changes from GFS v16.3.21 +* No changes from GFS v16.3.22 HPSS ARCHIVE ------------ -* No changes from GFS v16.3.21 +* No changes from GFS v16.3.22 JOB DEPENDENCIES AND FLOW DIAGRAM --------------------------------- -* No changes from GFS v16.3.21 +* No changes from GFS v16.3.22 DOCUMENTATION ------------- -* No changes from GFS v16.3.21 +* No changes from GFS v16.3.22 PREPARED BY ----------- -Kate.Friedman@noaa.gov -George.Gayno@noaa.gov -Andrew.Collard@noaa.gov +Ali Salimi Tarazouj +Jessica Meixner From 4a2c51b754ac24bfb3df45d10e0b7be15e0ae2f5 Mon Sep 17 00:00:00 2001 From: AliS-Noaa Date: Tue, 28 Jan 2025 19:57:09 +0000 Subject: [PATCH 10/22] minor revision in the Release_Notes.md --- docs/Release_Notes.md | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/docs/Release_Notes.md b/docs/Release_Notes.md index fea9c61614..bda414b216 100644 --- a/docs/Release_Notes.md +++ b/docs/Release_Notes.md @@ -49,7 +49,7 @@ cd ../ecf VERSION FILE CHANGES -------------------- -* No changes from GFS v16.3.22 +* No changes from GFS v16.3.21 SORC CHANGES ------------ @@ -59,7 +59,7 @@ SORC CHANGES JOBS CHANGES ------------ -* No changes from GFS v16.3.22 +* No changes from GFS v16.3.21 PARM/CONFIG CHANGES ------------------- @@ -74,12 +74,12 @@ SCRIPT CHANGES FIX CHANGES ----------- -* No changes from GFS v16.3.22 +* No changes from GFS v16.3.21 MODULE CHANGES -------------- -* No changes from GFS v16.3.22 +* No changes from GFS v16.3.21 CHANGES TO FILE AND FILE SIZES ------------------------------ @@ -106,24 +106,27 @@ PRE-IMPLEMENTATION TESTING REQUIREMENTS DISSEMINATION INFORMATION ------------------------- -* No changes from GFS v16.3.22 +* No changes from GFS v16.3.21 HPSS ARCHIVE ------------ -* No changes from GFS v16.3.22 +* No changes from GFS v16.3.21 JOB DEPENDENCIES AND FLOW DIAGRAM --------------------------------- -* No changes from GFS v16.3.22 +* No changes from GFS v16.3.21 DOCUMENTATION ------------- -* No changes from GFS v16.3.22 +* No changes from GFS v16.3.21 PREPARED BY ----------- -Ali Salimi Tarazouj -Jessica Meixner +Kate.Friedman@noaa.gov +George.Gayno@noaa.gov +Andrew.Collard@noaa.gov +Ali.Salmi@noaa.gov +Jessica.Meixner@noaa.gov From 25f6086ff446f2589f6853f1563c47687253a996 Mon Sep 17 00:00:00 2001 From: AliS-Noaa Date: Tue, 28 Jan 2025 20:16:49 +0000 Subject: [PATCH 11/22] another minor correction to the Release_Notes.md --- docs/Release_Notes.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/Release_Notes.md b/docs/Release_Notes.md index bda414b216..2e21456107 100644 --- a/docs/Release_Notes.md +++ b/docs/Release_Notes.md @@ -3,8 +3,10 @@ GFS V16.3.22 RELEASE NOTES ------- PRELUDE ------- - -The WW3 point output program (ww3_outp) is improved to process per-time-step point output to reduce the resources for the wave_post_pnt jobs. +The upstream OBSPROC package is updated to v1.3. Along with this are the following companion updates: +* workflow and UFS_UTILS package updates to use the new AFWA global snow file due to the hemispheric snow files being phased out +* updated GSI code and convinfo file for saildrone observations +* ww3_outp is improved for th wave point output IMPLEMENTATION INSTRUCTIONS --------------------------- @@ -128,5 +130,5 @@ PREPARED BY Kate.Friedman@noaa.gov George.Gayno@noaa.gov Andrew.Collard@noaa.gov -Ali.Salmi@noaa.gov Jessica.Meixner@noaa.gov +Ali.Salimi@noaa.gov From 53f8172e6116166e77726adc200b30c32a612480 Mon Sep 17 00:00:00 2001 From: AliS-Noaa <118747646+AliS-Noaa@users.noreply.github.com> Date: Tue, 28 Jan 2025 15:22:20 -0500 Subject: [PATCH 12/22] Update docs/Release_Notes.md Co-authored-by: Jessica Meixner --- docs/Release_Notes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Release_Notes.md b/docs/Release_Notes.md index 2e21456107..2fe0a62e49 100644 --- a/docs/Release_Notes.md +++ b/docs/Release_Notes.md @@ -6,7 +6,7 @@ PRELUDE The upstream OBSPROC package is updated to v1.3. Along with this are the following companion updates: * workflow and UFS_UTILS package updates to use the new AFWA global snow file due to the hemispheric snow files being phased out * updated GSI code and convinfo file for saildrone observations -* ww3_outp is improved for th wave point output +* ww3_outp is improved for the wave point output IMPLEMENTATION INSTRUCTIONS --------------------------- From 61e35778e1bdb00e366dd3815fb48c9e9f508b98 Mon Sep 17 00:00:00 2001 From: AliS-Noaa <118747646+AliS-Noaa@users.noreply.github.com> Date: Tue, 28 Jan 2025 15:26:41 -0500 Subject: [PATCH 13/22] Update docs/Release_Notes.md Co-authored-by: Jessica Meixner --- docs/Release_Notes.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/Release_Notes.md b/docs/Release_Notes.md index 2fe0a62e49..b91ae035e8 100644 --- a/docs/Release_Notes.md +++ b/docs/Release_Notes.md @@ -101,6 +101,7 @@ PRE-IMPLEMENTATION TESTING REQUIREMENTS --------------------------------------- * Which production jobs should be tested as part of this implementation? + * emcsfc_sfc_prep and analysis * wave_post_pnt * Does this change require a 30-day evaluation? * No From 0892513d2ac19548dadd5ca3eebab796c62eb074 Mon Sep 17 00:00:00 2001 From: AliS-Noaa <118747646+AliS-Noaa@users.noreply.github.com> Date: Tue, 28 Jan 2025 15:26:59 -0500 Subject: [PATCH 14/22] Update docs/Release_Notes.md Co-authored-by: Jessica Meixner --- docs/Release_Notes.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/Release_Notes.md b/docs/Release_Notes.md index b91ae035e8..eec01824f3 100644 --- a/docs/Release_Notes.md +++ b/docs/Release_Notes.md @@ -86,6 +86,13 @@ MODULE CHANGES CHANGES TO FILE AND FILE SIZES ------------------------------ +No longer ingest: +* `${RUN}.${cycle}.NPR.SNWN.SP.S1200.MESH16.grb` (`AFWA_NH_FILE`) +* `${RUN}.${cycle}.NPR.SNWS.SP.S1200.MESH16.grb` (`AFWA_SH_FILE`) + +Now ingest: +* `${RUN}.${cycle}.snow.usaf.grib2` (`AFWA_GLOBAL_FILE`) + No longer uses the parallel command file `cmdfile` for the `scripts/exgfs_wave_post_pnt.sh`. From b2220ccc2a99fe86a5d8ab6353d25b8090c3f885 Mon Sep 17 00:00:00 2001 From: AliS-Noaa <118747646+AliS-Noaa@users.noreply.github.com> Date: Tue, 28 Jan 2025 15:27:09 -0500 Subject: [PATCH 15/22] Update docs/Release_Notes.md Co-authored-by: Jessica Meixner --- docs/Release_Notes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Release_Notes.md b/docs/Release_Notes.md index eec01824f3..75e0d4bc59 100644 --- a/docs/Release_Notes.md +++ b/docs/Release_Notes.md @@ -76,7 +76,7 @@ SCRIPT CHANGES FIX CHANGES ----------- -* No changes from GFS v16.3.21 +* GSI `global_convinfo.txt` fix update for saildrone``` MODULE CHANGES -------------- From 52558acb824956c9b10f1920b58394659d23ce68 Mon Sep 17 00:00:00 2001 From: AliS-Noaa <118747646+AliS-Noaa@users.noreply.github.com> Date: Tue, 28 Jan 2025 15:27:16 -0500 Subject: [PATCH 16/22] Update docs/Release_Notes.md Co-authored-by: Jessica Meixner --- docs/Release_Notes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Release_Notes.md b/docs/Release_Notes.md index 75e0d4bc59..115e043e3f 100644 --- a/docs/Release_Notes.md +++ b/docs/Release_Notes.md @@ -51,7 +51,7 @@ cd ../ecf VERSION FILE CHANGES -------------------- -* No changes from GFS v16.3.21 +* * `versions/run.ver` - change `version=v16.3.22`, `gfs_ver=v16.3.22`, and `obsproc_ver=v1.3` SORC CHANGES ------------ From e22af6bb3405ea92da44677e40387787f2ca236e Mon Sep 17 00:00:00 2001 From: AliS-Noaa <118747646+AliS-Noaa@users.noreply.github.com> Date: Tue, 28 Jan 2025 15:27:23 -0500 Subject: [PATCH 17/22] Update docs/Release_Notes.md Co-authored-by: Jessica Meixner --- docs/Release_Notes.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/Release_Notes.md b/docs/Release_Notes.md index 115e043e3f..70b21907a5 100644 --- a/docs/Release_Notes.md +++ b/docs/Release_Notes.md @@ -56,7 +56,9 @@ VERSION FILE CHANGES SORC CHANGES ------------ -* `ww3_outp` program and associated scripts are improved to process the per-time-step point outputs more efficiently. +* New UFS_UTILS tag - `emcsfc_snow2mdl` program and associated scripts are updated to process global AFWA snow data +* New GSI tag - `src/gsi/read_prepbufr.f90` code update for new saildrone subtype +* New MODEL tag - WW3 program `ww3_outp` and associated scripts are improved to process the per-time-step point outputs more efficiently. JOBS CHANGES ------------ From aaf1a46ff326f5296084dc43ae045b6afb41128b Mon Sep 17 00:00:00 2001 From: AliS-Noaa <118747646+AliS-Noaa@users.noreply.github.com> Date: Tue, 28 Jan 2025 15:27:32 -0500 Subject: [PATCH 18/22] Update docs/Release_Notes.md Co-authored-by: Jessica Meixner --- docs/Release_Notes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Release_Notes.md b/docs/Release_Notes.md index 70b21907a5..cd7867ba01 100644 --- a/docs/Release_Notes.md +++ b/docs/Release_Notes.md @@ -63,7 +63,7 @@ SORC CHANGES JOBS CHANGES ------------ -* No changes from GFS v16.3.21 +* `jobs/JGLOBAL_ATMOS_EMCSFC_SFC_PREP` - new AFWA filename PARM/CONFIG CHANGES ------------------- From de078f6e4b995a13853a090a7e18f86cd6bf4d2d Mon Sep 17 00:00:00 2001 From: AliS-Noaa Date: Tue, 28 Jan 2025 22:11:43 +0000 Subject: [PATCH 19/22] adding two wave jobs for testing to the Release note --- docs/Release_Notes.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/Release_Notes.md b/docs/Release_Notes.md index cd7867ba01..f2d39e9439 100644 --- a/docs/Release_Notes.md +++ b/docs/Release_Notes.md @@ -112,6 +112,8 @@ PRE-IMPLEMENTATION TESTING REQUIREMENTS * Which production jobs should be tested as part of this implementation? * emcsfc_sfc_prep and analysis * wave_post_pnt + * wave_post_bndpnt + * wave_post_bndpntbll * Does this change require a 30-day evaluation? * No From 14bb68f6e42d85951abcbf208ff5b4cb6459abd9 Mon Sep 17 00:00:00 2001 From: AliS-Noaa Date: Wed, 29 Jan 2025 15:09:30 +0000 Subject: [PATCH 20/22] adding sperate log file for each wave post point jobs --- scripts/exgfs_wave_post_pnt.sh | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/scripts/exgfs_wave_post_pnt.sh b/scripts/exgfs_wave_post_pnt.sh index 26c139b141..c1a9ab9735 100755 --- a/scripts/exgfs_wave_post_pnt.sh +++ b/scripts/exgfs_wave_post_pnt.sh @@ -410,13 +410,13 @@ if [ ${CFP_MP:-"NO"} = "YES" ]; then if [ "$DOBNDPNT_WAV" = YES ]; then if [ "$DOSPC_WAV" = YES ]; then - echo "$USHwave/wave_tar.sh $WAV_MOD_TAG ibp $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 " >> cmdtarfile + echo "$USHwave/wave_tar.sh $WAV_MOD_TAG ibp $Nb > ${WAV_MOD_TAG}_ibp_tar.out 2>&1 " >> cmdtarfile nm=$(( nm + 1 )) fi if [ "$DOBLL_WAV" = YES ]; then - echo "$USHwave/wave_tar.sh $WAV_MOD_TAG ibpbull $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 " >> cmdtarfile + echo "$USHwave/wave_tar.sh $WAV_MOD_TAG ibpbull $Nb > ${WAV_MOD_TAG}_ibpbull_tar.out 2>&1 " >> cmdtarfile nm=$(( nm + 1 )) - echo "$USHwave/wave_tar.sh $WAV_MOD_TAG ibpcbull $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 " >> cmdtarfile + echo "$USHwave/wave_tar.sh $WAV_MOD_TAG ibpcbull $Nb > ${WAV_MOD_TAG}_ibpcbull_tar.out 2>&1 " >> cmdtarfile nm=$(( nm + 1 )) fi else @@ -425,28 +425,28 @@ nm=$(( nm + 1 )) fi if [ "$DOBLL_WAV" = YES ]; then - echo "$USHwave/wave_tar.sh $WAV_MOD_TAG bull $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 " >> cmdtarfile + echo "$USHwave/wave_tar.sh $WAV_MOD_TAG bull $Nb > ${WAV_MOD_TAG}_bull_tar.out 2>&1 " >> cmdtarfile nm=$(( nm + 1 )) - echo "$USHwave/wave_tar.sh $WAV_MOD_TAG cbull $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 " >> cmdtarfile + echo "$USHwave/wave_tar.sh $WAV_MOD_TAG cbull $Nb > ${WAV_MOD_TAG}_cbull_tar.out 2>&1 " >> cmdtarfile nm=$(( nm + 1 )) fi fi else if [ "$DOBNDPNT_WAV" = YES ]; then if [ "$DOSPC_WAV" = YES ]; then - echo "$USHwave/wave_tar.sh $WAV_MOD_TAG ibp $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 " >> cmdtarfile + echo "$USHwave/wave_tar.sh $WAV_MOD_TAG ibp $Nb > ${WAV_MOD_TAG}_ibp_tar.out 2>&1 " >> cmdtarfile fi if [ "$DOBLL_WAV" = YES ]; then - echo "$USHwave/wave_tar.sh $WAV_MOD_TAG ibpbull $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 " >> cmdtarfile - echo "$USHwave/wave_tar.sh $WAV_MOD_TAG ibpcbull $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 " >> cmdtarfile + echo "$USHwave/wave_tar.sh $WAV_MOD_TAG ibpbull $Nb > ${WAV_MOD_TAG}_ibpbull_tar.out 2>&1 " >> cmdtarfile + echo "$USHwave/wave_tar.sh $WAV_MOD_TAG ibpcbull $Nb > ${WAV_MOD_TAG}_ibpcbull_tar.out 2>&1 " >> cmdtarfile fi else if [ "$DOSPC_WAV" = YES ]; then echo "$USHwave/wave_tar.sh $WAV_MOD_TAG spec $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 " >> cmdtarfile fi if [ "$DOBLL_WAV" = YES ]; then - echo "$USHwave/wave_tar.sh $WAV_MOD_TAG bull $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 " >> cmdtarfile - echo "$USHwave/wave_tar.sh $WAV_MOD_TAG cbull $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 " >> cmdtarfile + echo "$USHwave/wave_tar.sh $WAV_MOD_TAG bull $Nb > ${WAV_MOD_TAG}_bull_tar.out 2>&1 " >> cmdtarfile + echo "$USHwave/wave_tar.sh $WAV_MOD_TAG cbull $Nb > ${WAV_MOD_TAG}_cbull_tar.out 2>&1 " >> cmdtarfile fi fi fi From 9ae2e741a77461e4c6d82d613bf3933048dbb1dc Mon Sep 17 00:00:00 2001 From: AliS-Noaa Date: Wed, 29 Jan 2025 16:32:37 +0000 Subject: [PATCH 21/22] adding the tag to checkout.sh and also some minor revision of the release note --- docs/Release_Notes.md | 18 +++++++++--------- sorc/checkout.sh | 3 +-- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/docs/Release_Notes.md b/docs/Release_Notes.md index f2d39e9439..2c5656ae73 100644 --- a/docs/Release_Notes.md +++ b/docs/Release_Notes.md @@ -51,7 +51,7 @@ cd ../ecf VERSION FILE CHANGES -------------------- -* * `versions/run.ver` - change `version=v16.3.22`, `gfs_ver=v16.3.22`, and `obsproc_ver=v1.3` +* `versions/run.ver` - change `version=v16.3.22`, `gfs_ver=v16.3.22`, and `obsproc_ver=v1.3` SORC CHANGES ------------ @@ -68,17 +68,20 @@ JOBS CHANGES PARM/CONFIG CHANGES ------------------- -* `config.resources.emc.dyn` and `config.resources.nco.static` are changed to reduce the resources accordingly. +In `config.resources.emc.dyn` and `config.resources.nco.static` following resources are changed: +* for wavepostbndpnt: npe from 240 to 1; wtime from 1hr to 30min +* for wavepostbndpntbll: npe from 448 to 2; wtime from 1hr to 10min +* for wavepostpnt: npe from 200 to 3; wtime from 1.5hr to 35min SCRIPT CHANGES -------------- -* `scripts/exgfs_wave_post_pnt.sh` is changed to create be compatible with the new `ww3_outp`. +* `scripts/exgfs_wave_post_pnt.sh` is changed to be compatible with the new `ww3_outp`. FIX CHANGES ----------- -* GSI `global_convinfo.txt` fix update for saildrone``` +* GSI `global_convinfo.txt` fix update for saildrone MODULE CHANGES -------------- @@ -89,15 +92,12 @@ CHANGES TO FILE AND FILE SIZES ------------------------------ No longer ingest: -* `${RUN}.${cycle}.NPR.SNWN.SP.S1200.MESH16.grb` (`AFWA_NH_FILE`) +* `${RUN}.${cycle}.NPR.SNWN.SP.S1200.MESH16.grb` (`AFWA_NH_FILE`) * `${RUN}.${cycle}.NPR.SNWS.SP.S1200.MESH16.grb` (`AFWA_SH_FILE`) -Now ingest: +Now ingest: * `${RUN}.${cycle}.snow.usaf.grib2` (`AFWA_GLOBAL_FILE`) -No longer uses the parallel command file `cmdfile` for the `scripts/exgfs_wave_post_pnt.sh`. - - ENVIRONMENT AND RESOURCE CHANGES -------------------------------- diff --git a/sorc/checkout.sh b/sorc/checkout.sh index b6e6960f7a..56067ee5f2 100755 --- a/sorc/checkout.sh +++ b/sorc/checkout.sh @@ -23,8 +23,7 @@ echo $topdir echo fv3gfs checkout ... if [[ ! -d fv3gfs.fd ]] ; then rm -f ${topdir}/checkout-fv3gfs.log - git clone --recursive --branch prodpoint https://github.com/AliS-Noaa/ufs-weather-model.git fv3gfs.fd >> ${topdir}/checkout-fv3gfs.log 2>&1 - #git clone --recursive --branch GFS.v16.3.1 https://github.com/ufs-community/ufs-weather-model.git fv3gfs.fd >> ${topdir}/checkout-fv3gfs.log 2>&1 + git clone --recursive --branch GFS.v16.3.22 https://github.com/ufs-community/ufs-weather-model.git fv3gfs.fd >> ${topdir}/checkout-fv3gfs.log 2>&1 cd ${topdir} else echo 'Skip. Directory fv3gfs.fd already exists.' From 0c583d8ecb47f6cf468e03a14ad90a0d2751e77d Mon Sep 17 00:00:00 2001 From: AliS-Noaa Date: Thu, 30 Jan 2025 17:57:25 +0000 Subject: [PATCH 22/22] updating External.cfg with the correct ufs tag --- Externals.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Externals.cfg b/Externals.cfg index c24b9155fa..efb7a686e6 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -1,7 +1,7 @@ # External sub-modules of global-workflow [FV3GFS] -tag = GFS.v16.3.1 +tag = GFS.v16.3.22 local_path = sorc/fv3gfs.fd repo_url = https://github.com/ufs-community/ufs-weather-model.git protocol = git