diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000..1dc8f4ed61 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,5 @@ +Dockerfile +.git +.gitignore +tests/*.log +tests/log_ut_linux.gnu diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000000..5301085819 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,121 @@ +name: Pull Request Tests + +on: + push: + branches: + - develop + pull_request: + branches: + - develop + +jobs: + setup: + name: Set up + runs-on: ubuntu-latest + + outputs: + tn: ${{ steps.parse.outputs.tn }} + bld: ${{ steps.parse.outputs.bld }} + test: ${{ steps.parse.outputs.test }} + img: ${{ steps.parse.outputs.img }} + + steps: + - name: Checkout codes + uses: actions/checkout@v2 + + - name: Parse cases + id: parse + run: | + cd ${GITHUB_WORKSPACE}/tests/ci + parsed_output=( $(./parse.sh) ) + name_=${parsed_output[0]} + bld_=${parsed_output[1]} + test_=${parsed_output[2]} + img_=${parsed_output[3]} + + echo "::set-output name=tn::$name_" + echo "::set-output name=bld::$bld_" + echo "::set-output name=test::$test_" + echo "::set-output name=img::$img_" + + echo "test name : $name_" + echo "build set : $bld_" + echo "test set : $test_" + echo "image name: $img_" + + build: + name: Build (${{ matrix.bld_set }}) + needs: setup + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.setup.outputs.bld) }} + + steps: + - name: Checkout codes + uses: actions/checkout@v2 + with: + submodules: recursive + + - name: Build + run: | + printf '{\n "experimental": true\n}' | sudo tee /etc/docker/daemon.json >/dev/null + sudo systemctl restart docker + sleep 10 + cd tests/ci && ./ci.sh -n ${{ needs.setup.outputs.tn }} -b ${{ matrix.bld_set }} + + - name: Free up disk space + run: | + sudo docker rmi $(sudo docker image ls | grep -E -m1 '' | awk '{ print $3 }') + sudo docker rmi $(sudo docker image ls | awk '/ci-test-base/ { print $3 }') + + - name: Prepare artifacts + run: | + cd tests/ci + sudo docker save ${{ needs.setup.outputs.img }} | gzip >${{ needs.setup.outputs.img }}.tar.gz + tar cvjf artifact.tar.bz2 ${{ needs.setup.outputs.img }}.tar.gz ci.sh ci.test + + - name: Upload artifacts + uses: actions/upload-artifact@v2 + with: + name: ${{ matrix.bld_set }}.artifact.tar.bz2 + path: tests/ci/artifact.tar.bz2 + + utest: + name: Unit test (${{ needs.setup.outputs.tn }}, ${{ matrix.test_set }}) + needs: [setup,build] + runs-on: ubuntu-latest + #runs-on: self-hosted + + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.setup.outputs.test) }} + + steps: + - name: Download artifacts + uses: actions/download-artifact@v2 + with: + name: ${{ matrix.artifact }}.artifact.tar.bz2 + + - name: Prepare artifacts + run: | + tar xvjf artifact.tar.bz2 && rm -f artifact.tar.bz2 + sudo docker load --input ${{ needs.setup.outputs.img }}.tar.gz && rm -f ${{ needs.setup.outputs.img }}.tar.gz + + - name: Run utest + run: ./ci.sh -n ${{ needs.setup.outputs.tn }} -r ${{ matrix.test_set }} + + - name: Upload memory usage file + if: ${{ always() }} + uses: actions/upload-artifact@v2 + with: + name: memory_stat_${{ matrix.test_set }} + path: memory_stat + + - name: Clean up + if: ${{ always() }} + run: | + rm -f ci.sh ci.test + sudo docker rm my-container && sudo docker rmi ${{ needs.setup.outputs.img }}:latest + sudo docker volume rm DataVolume diff --git a/.github/workflows/manage.yml b/.github/workflows/manage.yml new file mode 100644 index 0000000000..9f12a8f372 --- /dev/null +++ b/.github/workflows/manage.yml @@ -0,0 +1,54 @@ +name: Manage workflows + +on: + workflow_run: + workflows: ["Pull Request Tests"] + types: + - requested + +jobs: + job1: + name: Job 1 + runs-on: ubuntu-latest + + steps: + - name: Checkout codes + uses: actions/checkout@v2 + + - name: Check if skip-ci is requested + run: | + cd ${GITHUB_WORKSPACE}/tests/ci + repo="${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/actions/runs" + tr_id=$(cat ${GITHUB_EVENT_PATH} | ./json_helper.py get_trigger_id) + tr_br=$(cat ${GITHUB_EVENT_PATH} | ./json_helper.py get_trigger_br) + check=$(cat ${GITHUB_EVENT_PATH} | ./json_helper.py check_skip) + echo "::set-env name=TRIGGER_ID::${tr_id}" + echo "::set-env name=TRIGGER_BR::${tr_br}" + echo "skip-ci: ${check}" + if [[ $check == yes ]]; then + echo "skip-ci is requested" + echo '::set-env name=CURR_JOB::cancelled' + curl -X POST -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -H "Accept: application/vnd.github.v3+json" ${repo}/$tr_id/cancel + else + echo '::set-env name=CURR_JOB::running' + fi + + - name: Cancel redundant jobs + run: | + echo "CURR_JOB is $CURR_JOB" + echo "TRIGGER_ID is $TRIGGER_ID" + echo "TRIGGER_BR is $TRIGGER_BR" + export GITHUB_ACTOR + export GITHUB_RUN_ID + export TRIGGER_ID + export TRIGGER_BR + cd ${GITHUB_WORKSPACE}/tests/ci + repo="${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/actions/runs" + cancel_ids=$(curl -H "Accept: application/vnd.github.v3+json" ${repo} | ./json_helper.py cancel_workflow) + echo "cancel ids: $cancel_ids" + if [[ $cancel_ids != '' ]]; then + for i in $cancel_ids; do + curl -X POST -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -H "Accept: application/vnd.github.v3+json" ${repo}/$i/cancel + done + fi + if: ${{ env.CURR_JOB == 'running' }} diff --git a/modulefiles/linux.gnu/fv3 b/modulefiles/linux.gnu/fv3 index f102049211..64d00092ef 100755 --- a/modulefiles/linux.gnu/fv3 +++ b/modulefiles/linux.gnu/fv3 @@ -15,6 +15,19 @@ export F77=${F77:-mpif77} export F90=${F90:-mpif90} export FC=${FC:-mpif90} +## +## set up variables for ../cmake/configure_linux.gnu.cmake +## +export CMAKE_Platform=linux.gnu +export CMAKE_C_COMPILER=${CC} +export CMAKE_CXX_COMPILER=${CXX} +export CMAKE_Fortran_COMPILER=${FC} + +## +## use own NetCDF library +## +export NETCDF=${NETCDF:-/usr/local} + ## ## use SIONlib library if installed and environment variable is set ## @@ -24,3 +37,19 @@ if [ ! "x$SIONLIB" == "x" ]; then export SIONLIB_INC="-I${SIONLIB}/include -I${SIONLIB}/include/mod_64" export SIONLIB_LIB="-L${SIONLIB}/lib -lsionmpi_f90_64 -lsionser_f90_64 -lsionmpi_64 -lsiongen_64 -lsionser_64 -lsioncom_64 -lsioncom_64_lock_none" fi + +## +## use pre-compiled EMSF library for above compiler / MPI combination +## +export ESMFMKFILE=${ESMFMKFILE:-/usr/local/lib/esmf.mk} + +## +## NCEP libraries (need to download and build manually, see doc/README_{UBUNTU,CENTOS,...}.txt and https://github.com/NCAR/NCEPlibs) +## +export NCEPLIBS_DIR=${NCEPLIBS_DIR:-/usr/local/NCEPlibs} +export bacio_DIR=${NCEPLIBS_DIR}/bacio-2.4.0 +export nemsio_DIR=${NCEPLIBS_DIR}/nemsio-2.5.1 +export w3nco_DIR=${NCEPLIBS_DIR}/w3nco-2.4.0 +export sp_DIR=${NCEPLIBS_DIR}/sp-2.3.0 +export w3emc_DIR=${NCEPLIBS_DIR}/w3emc-2.7.0 +export sigio_DIR=${NCEPLIBS_DIR}/sigio-2.3.0 diff --git a/tests/RegressionTests_hera.intel.log b/tests/RegressionTests_hera.intel.log index a89561c1ed..d97ad8e04a 100644 --- a/tests/RegressionTests_hera.intel.log +++ b/tests/RegressionTests_hera.intel.log @@ -1,9 +1,9 @@ -Wed Sep 2 15:01:05 UTC 2020 +Tue Sep 8 20:00:20 UTC 2020 Start Regression test baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_control_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_33277/fv3_ccpp_control_prod +working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_RT/rt_300055/fv3_ccpp_control_prod Checking test 001 fv3_ccpp_control results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -71,7 +71,7 @@ Test 001 fv3_ccpp_control PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_control_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_33277/fv3_ccpp_decomp_prod +working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_RT/rt_300055/fv3_ccpp_decomp_prod Checking test 002 fv3_ccpp_decomp results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -139,7 +139,7 @@ Test 002 fv3_ccpp_decomp PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_control_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_33277/fv3_ccpp_2threads_prod +working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_RT/rt_300055/fv3_ccpp_2threads_prod Checking test 003 fv3_ccpp_2threads results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -207,7 +207,7 @@ Test 003 fv3_ccpp_2threads PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_restart_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_33277/fv3_ccpp_restart_prod +working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_RT/rt_300055/fv3_ccpp_restart_prod Checking test 004 fv3_ccpp_restart results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -275,7 +275,7 @@ Test 004 fv3_ccpp_restart PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_read_inc_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_33277/fv3_ccpp_read_inc_prod +working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_RT/rt_300055/fv3_ccpp_read_inc_prod Checking test 005 fv3_ccpp_read_inc results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -343,7 +343,7 @@ Test 005 fv3_ccpp_read_inc PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_wrtGauss_netcdf_esmf_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_33277/fv3_ccpp_wrtGauss_netcdf_esmf_prod +working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_RT/rt_300055/fv3_ccpp_wrtGauss_netcdf_esmf_prod Checking test 006 fv3_ccpp_wrtGauss_netcdf_esmf results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -391,7 +391,7 @@ Test 006 fv3_ccpp_wrtGauss_netcdf_esmf PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_wrtGauss_netcdf_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_33277/fv3_ccpp_wrtGauss_netcdf_prod +working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_RT/rt_300055/fv3_ccpp_wrtGauss_netcdf_prod Checking test 007 fv3_ccpp_wrtGauss_netcdf results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -439,7 +439,7 @@ Test 007 fv3_ccpp_wrtGauss_netcdf PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_wrtGlatlon_netcdf_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_33277/fv3_ccpp_wrtGlatlon_netcdf_prod +working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_RT/rt_300055/fv3_ccpp_wrtGlatlon_netcdf_prod Checking test 008 fv3_ccpp_wrtGlatlon_netcdf results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -487,7 +487,7 @@ Test 008 fv3_ccpp_wrtGlatlon_netcdf PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_wrtGauss_nemsio_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_33277/fv3_ccpp_wrtGauss_nemsio_prod +working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_RT/rt_300055/fv3_ccpp_wrtGauss_nemsio_prod Checking test 009 fv3_ccpp_wrtGauss_nemsio results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -535,7 +535,7 @@ Test 009 fv3_ccpp_wrtGauss_nemsio PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_wrtGauss_nemsio_c192_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_33277/fv3_ccpp_wrtGauss_nemsio_c192_prod +working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_RT/rt_300055/fv3_ccpp_wrtGauss_nemsio_c192_prod Checking test 010 fv3_ccpp_wrtGauss_nemsio_c192 results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -583,7 +583,7 @@ Test 010 fv3_ccpp_wrtGauss_nemsio_c192 PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_stochy_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_33277/fv3_ccpp_stochy_prod +working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_RT/rt_300055/fv3_ccpp_stochy_prod Checking test 011 fv3_ccpp_stochy results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -651,7 +651,7 @@ Test 011 fv3_ccpp_stochy PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_iau_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_33277/fv3_ccpp_iau_prod +working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_RT/rt_300055/fv3_ccpp_iau_prod Checking test 012 fv3_ccpp_iau results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -719,7 +719,7 @@ Test 012 fv3_ccpp_iau PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_ca_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_33277/fv3_ccpp_ca_prod +working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_RT/rt_300055/fv3_ccpp_ca_prod Checking test 013 fv3_ccpp_ca results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -787,7 +787,7 @@ Test 013 fv3_ccpp_ca PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_lndp_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_33277/fv3_ccpp_lndp_prod +working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_RT/rt_300055/fv3_ccpp_lndp_prod Checking test 014 fv3_ccpp_lndp results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -855,7 +855,7 @@ Test 014 fv3_ccpp_lndp PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_lheatstrg_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_33277/fv3_ccpp_lheatstrg_prod +working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_RT/rt_300055/fv3_ccpp_lheatstrg_prod Checking test 015 fv3_ccpp_lheatstrg results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -903,7 +903,7 @@ Test 015 fv3_ccpp_lheatstrg PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_gfdlmprad_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_33277/fv3_ccpp_gfdlmprad_prod +working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_RT/rt_300055/fv3_ccpp_gfdlmprad_prod Checking test 016 fv3_ccpp_gfdlmprad results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -952,7 +952,7 @@ Test 016 fv3_ccpp_gfdlmprad PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_gfdlmprad_atmwav_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_33277/fv3_ccpp_gfdlmprad_atmwav_prod +working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_RT/rt_300055/fv3_ccpp_gfdlmprad_atmwav_prod Checking test 017 fv3_ccpp_gfdlmprad_atmwav results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1000,9 +1000,8 @@ Checking test 017 fv3_ccpp_gfdlmprad_atmwav results .... Test 017 fv3_ccpp_gfdlmprad_atmwav PASS -# DH* manually copied over the results for the following test from a second regression test run because of file system issues on hera baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_wrtGauss_nemsio_c768_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_93431/fv3_ccpp_wrtGauss_nemsio_c768_prod +working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_RT/rt_300055/fv3_ccpp_wrtGauss_nemsio_c768_prod Checking test 018 fv3_ccpp_wrtGauss_nemsio_c768 results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1048,11 +1047,10 @@ Checking test 018 fv3_ccpp_wrtGauss_nemsio_c768 results .... Comparing out_grd.ant_9km .........OK Comparing out_grd.aoc_9km .........OK Test 018 fv3_ccpp_wrtGauss_nemsio_c768 PASS -# *DH baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_multigases_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_33277/fv3_ccpp_multigases_prod +working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_RT/rt_300055/fv3_ccpp_multigases_prod Checking test 019 fv3_ccpp_multigases results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1126,7 +1124,7 @@ Test 019 fv3_ccpp_multigases PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_control_32bit_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_33277/fv3_ccpp_control_32bit_prod +working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_RT/rt_300055/fv3_ccpp_control_32bit_prod Checking test 020 fv3_ccpp_control_32bit results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1194,7 +1192,7 @@ Test 020 fv3_ccpp_control_32bit PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_stretched_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_33277/fv3_ccpp_stretched_prod +working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_RT/rt_300055/fv3_ccpp_stretched_prod Checking test 021 fv3_ccpp_stretched results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1250,7 +1248,7 @@ Test 021 fv3_ccpp_stretched PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_stretched_nest_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_33277/fv3_ccpp_stretched_nest_prod +working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_RT/rt_300055/fv3_ccpp_stretched_nest_prod Checking test 022 fv3_ccpp_stretched_nest results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1317,7 +1315,7 @@ Test 022 fv3_ccpp_stretched_nest PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_regional_control_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_33277/fv3_ccpp_regional_control_prod +working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_RT/rt_300055/fv3_ccpp_regional_control_prod Checking test 023 fv3_ccpp_regional_control results .... Comparing atmos_4xdaily.nc .........OK Comparing fv3_history2d.nc .........OK @@ -1328,7 +1326,7 @@ Test 023 fv3_ccpp_regional_control PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_regional_restart_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_33277/fv3_ccpp_regional_restart_prod +working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_RT/rt_300055/fv3_ccpp_regional_restart_prod Checking test 024 fv3_ccpp_regional_restart results .... Comparing atmos_4xdaily.nc .........OK Comparing fv3_history2d.nc .........OK @@ -1337,7 +1335,7 @@ Test 024 fv3_ccpp_regional_restart PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_regional_quilt_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_33277/fv3_ccpp_regional_quilt_prod +working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_RT/rt_300055/fv3_ccpp_regional_quilt_prod Checking test 025 fv3_ccpp_regional_quilt results .... Comparing atmos_4xdaily.nc .........OK Comparing dynf000.nc .........OK @@ -1348,7 +1346,7 @@ Test 025 fv3_ccpp_regional_quilt PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_regional_c768_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_33277/fv3_ccpp_regional_c768_prod +working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_RT/rt_300055/fv3_ccpp_regional_c768_prod Checking test 026 fv3_ccpp_regional_c768 results .... Comparing atmos_4xdaily.nc .........OK Comparing dynf000.nc .........OK @@ -1359,19 +1357,19 @@ Test 026 fv3_ccpp_regional_c768 PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_control_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_33277/fv3_ccpp_control_debug_prod +working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_RT/rt_300055/fv3_ccpp_control_debug_prod Checking test 027 fv3_ccpp_control_debug results .... Test 027 fv3_ccpp_control_debug PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_stretched_nest_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_33277/fv3_ccpp_stretched_nest_debug_prod +working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_RT/rt_300055/fv3_ccpp_stretched_nest_debug_prod Checking test 028 fv3_ccpp_stretched_nest_debug results .... Test 028 fv3_ccpp_stretched_nest_debug PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_gfdlmp_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_33277/fv3_ccpp_gfdlmp_prod +working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_RT/rt_300055/fv3_ccpp_gfdlmp_prod Checking test 029 fv3_ccpp_gfdlmp results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1419,7 +1417,7 @@ Test 029 fv3_ccpp_gfdlmp PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_gfdlmprad_gwd_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_33277/fv3_ccpp_gfdlmprad_gwd_prod +working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_RT/rt_300055/fv3_ccpp_gfdlmprad_gwd_prod Checking test 030 fv3_ccpp_gfdlmprad_gwd results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1467,7 +1465,7 @@ Test 030 fv3_ccpp_gfdlmprad_gwd PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_gfdlmprad_noahmp_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_33277/fv3_ccpp_gfdlmprad_noahmp_prod +working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_RT/rt_300055/fv3_ccpp_gfdlmprad_noahmp_prod Checking test 031 fv3_ccpp_gfdlmprad_noahmp results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1515,7 +1513,7 @@ Test 031 fv3_ccpp_gfdlmprad_noahmp PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_gfdlmp_hwrfsas_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_33277/fv3_ccpp_gfdlmp_hwrfsas_prod +working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_RT/rt_300055/fv3_ccpp_gfdlmp_hwrfsas_prod Checking test 032 fv3_ccpp_gfdlmp_hwrfsas results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1563,7 +1561,7 @@ Test 032 fv3_ccpp_gfdlmp_hwrfsas PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_csawmg_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_33277/fv3_ccpp_csawmg_prod +working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_RT/rt_300055/fv3_ccpp_csawmg_prod Checking test 033 fv3_ccpp_csawmg results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1611,7 +1609,7 @@ Test 033 fv3_ccpp_csawmg PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_satmedmf_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_33277/fv3_ccpp_satmedmf_prod +working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_RT/rt_300055/fv3_ccpp_satmedmf_prod Checking test 034 fv3_ccpp_satmedmf results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1659,7 +1657,7 @@ Test 034 fv3_ccpp_satmedmf PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_satmedmfq_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_33277/fv3_ccpp_satmedmfq_prod +working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_RT/rt_300055/fv3_ccpp_satmedmfq_prod Checking test 035 fv3_ccpp_satmedmfq results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1707,7 +1705,7 @@ Test 035 fv3_ccpp_satmedmfq PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_gfdlmp_32bit_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_33277/fv3_ccpp_gfdlmp_32bit_prod +working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_RT/rt_300055/fv3_ccpp_gfdlmp_32bit_prod Checking test 036 fv3_ccpp_gfdlmp_32bit results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1755,7 +1753,7 @@ Test 036 fv3_ccpp_gfdlmp_32bit PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_gfdlmprad_32bit_post_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_33277/fv3_ccpp_gfdlmprad_32bit_post_prod +working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_RT/rt_300055/fv3_ccpp_gfdlmprad_32bit_post_prod Checking test 037 fv3_ccpp_gfdlmprad_32bit_post results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1807,7 +1805,7 @@ Test 037 fv3_ccpp_gfdlmprad_32bit_post PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_cpt_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_33277/fv3_ccpp_cpt_prod +working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_RT/rt_300055/fv3_ccpp_cpt_prod Checking test 038 fv3_ccpp_cpt results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1861,7 +1859,7 @@ Test 038 fv3_ccpp_cpt PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_gsd_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_33277/fv3_ccpp_gsd_prod +working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_RT/rt_300055/fv3_ccpp_gsd_prod Checking test 039 fv3_ccpp_gsd results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1953,7 +1951,7 @@ Test 039 fv3_ccpp_gsd PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_rap_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_33277/fv3_ccpp_rap_prod +working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_RT/rt_300055/fv3_ccpp_rap_prod Checking test 040 fv3_ccpp_rap results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2021,7 +2019,7 @@ Test 040 fv3_ccpp_rap PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_hrrr_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_33277/fv3_ccpp_hrrr_prod +working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_RT/rt_300055/fv3_ccpp_hrrr_prod Checking test 041 fv3_ccpp_hrrr results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2089,7 +2087,7 @@ Test 041 fv3_ccpp_hrrr PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_thompson_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_33277/fv3_ccpp_thompson_prod +working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_RT/rt_300055/fv3_ccpp_thompson_prod Checking test 042 fv3_ccpp_thompson results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2157,7 +2155,7 @@ Test 042 fv3_ccpp_thompson PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_thompson_no_aero_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_33277/fv3_ccpp_thompson_no_aero_prod +working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_RT/rt_300055/fv3_ccpp_thompson_no_aero_prod Checking test 043 fv3_ccpp_thompson_no_aero results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2225,7 +2223,7 @@ Test 043 fv3_ccpp_thompson_no_aero PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_rrfs_v1beta_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_33277/fv3_ccpp_rrfs_v1beta_prod +working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_RT/rt_300055/fv3_ccpp_rrfs_v1beta_prod Checking test 044 fv3_ccpp_rrfs_v1beta results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2317,7 +2315,7 @@ Test 044 fv3_ccpp_rrfs_v1beta PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_gfs_v15p2_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_33277/fv3_ccpp_gfs_v15p2_prod +working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_RT/rt_300055/fv3_ccpp_gfs_v15p2_prod Checking test 045 fv3_ccpp_gfs_v15p2 results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2385,7 +2383,7 @@ Test 045 fv3_ccpp_gfs_v15p2 PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_gfs_v16beta_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_33277/fv3_ccpp_gfs_v16beta_prod +working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_RT/rt_300055/fv3_ccpp_gfs_v16beta_prod Checking test 046 fv3_ccpp_gfs_v16beta results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2453,7 +2451,7 @@ Test 046 fv3_ccpp_gfs_v16beta PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_gfs_v15p2_RRTMGP_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_33277/fv3_ccpp_gfs_v15p2_RRTMGP_prod +working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_RT/rt_300055/fv3_ccpp_gfs_v15p2_RRTMGP_prod Checking test 047 fv3_ccpp_gfs_v15p2_RRTMGP results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2521,7 +2519,7 @@ Test 047 fv3_ccpp_gfs_v15p2_RRTMGP PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_gfs_v16beta_RRTMGP_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_33277/fv3_ccpp_gfs_v16beta_RRTMGP_prod +working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_RT/rt_300055/fv3_ccpp_gfs_v16beta_RRTMGP_prod Checking test 048 fv3_ccpp_gfs_v16beta_RRTMGP results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2589,7 +2587,7 @@ Test 048 fv3_ccpp_gfs_v16beta_RRTMGP PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_gfsv16_csawmg_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_33277/fv3_ccpp_gfsv16_csawmg_prod +working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_RT/rt_300055/fv3_ccpp_gfsv16_csawmg_prod Checking test 049 fv3_ccpp_gfsv16_csawmg results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2637,7 +2635,7 @@ Test 049 fv3_ccpp_gfsv16_csawmg PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_gfsv16_csawmgt_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_33277/fv3_ccpp_gfsv16_csawmgt_prod +working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_RT/rt_300055/fv3_ccpp_gfsv16_csawmgt_prod Checking test 050 fv3_ccpp_gfsv16_csawmgt results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2685,7 +2683,7 @@ Test 050 fv3_ccpp_gfsv16_csawmgt PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_gocart_clm_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_33277/fv3_ccpp_gocart_clm_prod +working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_RT/rt_300055/fv3_ccpp_gocart_clm_prod Checking test 051 fv3_ccpp_gocart_clm results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2733,7 +2731,7 @@ Test 051 fv3_ccpp_gocart_clm PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_gfs_v16beta_flake_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_33277/fv3_ccpp_gfs_v16beta_flake_prod +working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_RT/rt_300055/fv3_ccpp_gfs_v16beta_flake_prod Checking test 052 fv3_ccpp_gfs_v16beta_flake results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2801,7 +2799,7 @@ Test 052 fv3_ccpp_gfs_v16beta_flake PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_gfs_v15p2_debug_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_33277/fv3_ccpp_gfs_v15p2_debug_prod +working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_RT/rt_300055/fv3_ccpp_gfs_v15p2_debug_prod Checking test 053 fv3_ccpp_gfs_v15p2_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2869,7 +2867,7 @@ Test 053 fv3_ccpp_gfs_v15p2_debug PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_gfs_v16beta_debug_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_33277/fv3_ccpp_gfs_v16beta_debug_prod +working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_RT/rt_300055/fv3_ccpp_gfs_v16beta_debug_prod Checking test 054 fv3_ccpp_gfs_v16beta_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2937,7 +2935,7 @@ Test 054 fv3_ccpp_gfs_v16beta_debug PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_gfs_v15p2_RRTMGP_debug_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_33277/fv3_ccpp_gfs_v15p2_RRTMGP_debug_prod +working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_RT/rt_300055/fv3_ccpp_gfs_v15p2_RRTMGP_debug_prod Checking test 055 fv3_ccpp_gfs_v15p2_RRTMGP_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -3005,7 +3003,7 @@ Test 055 fv3_ccpp_gfs_v15p2_RRTMGP_debug PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_gfs_v16beta_RRTMGP_debug_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_33277/fv3_ccpp_gfs_v16beta_RRTMGP_debug_prod +working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_RT/rt_300055/fv3_ccpp_gfs_v16beta_RRTMGP_debug_prod Checking test 056 fv3_ccpp_gfs_v16beta_RRTMGP_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -3073,7 +3071,7 @@ Test 056 fv3_ccpp_gfs_v16beta_RRTMGP_debug PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_gsd_debug_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_33277/fv3_ccpp_gsd_debug_prod +working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_RT/rt_300055/fv3_ccpp_gsd_debug_prod Checking test 057 fv3_ccpp_gsd_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -3141,7 +3139,7 @@ Test 057 fv3_ccpp_gsd_debug PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_gsd_diag3d_debug_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_33277/fv3_ccpp_gsd_diag3d_debug_prod +working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_RT/rt_300055/fv3_ccpp_gsd_diag3d_debug_prod Checking test 058 fv3_ccpp_gsd_diag3d_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -3209,7 +3207,7 @@ Test 058 fv3_ccpp_gsd_diag3d_debug PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_thompson_debug_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_33277/fv3_ccpp_thompson_debug_prod +working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_RT/rt_300055/fv3_ccpp_thompson_debug_prod Checking test 059 fv3_ccpp_thompson_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -3277,7 +3275,7 @@ Test 059 fv3_ccpp_thompson_debug PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_thompson_no_aero_debug_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_33277/fv3_ccpp_thompson_no_aero_debug_prod +working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_RT/rt_300055/fv3_ccpp_thompson_no_aero_debug_prod Checking test 060 fv3_ccpp_thompson_no_aero_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -3345,7 +3343,7 @@ Test 060 fv3_ccpp_thompson_no_aero_debug PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_rrfs_v1beta_debug_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_33277/fv3_ccpp_rrfs_v1beta_debug_prod +working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_RT/rt_300055/fv3_ccpp_rrfs_v1beta_debug_prod Checking test 061 fv3_ccpp_rrfs_v1beta_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -3411,8 +3409,7 @@ Checking test 061 fv3_ccpp_rrfs_v1beta_debug results .... Comparing RESTART/sfc_data.tile6.nc .........OK Test 061 fv3_ccpp_rrfs_v1beta_debug PASS -# DH* note: reran the previously failed test fv3_ccpp_wrtGauss_nemsio_c768_prod and manually updated this log file (file system issues on hera) + REGRESSION TEST WAS SUCCESSFUL -# *DH -Wed Sep 2 15:25:17 UTC 2020 -Elapsed time: 00h:24m:13s. Have a nice day! +Tue Sep 8 20:35:17 UTC 2020 +Elapsed time: 00h:34m:58s. Have a nice day! diff --git a/tests/RegressionTests_orion.intel.log b/tests/RegressionTests_orion.intel.log index b0f570e2ab..24aaf471d2 100644 --- a/tests/RegressionTests_orion.intel.log +++ b/tests/RegressionTests_orion.intel.log @@ -1,9 +1,9 @@ -Tue Sep 1 14:54:07 CDT 2020 +Tue Sep 8 15:13:05 CDT 2020 Start Regression test baseline dir = /work/noaa/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_control_ccpp -working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_381876/fv3_ccpp_control_prod +working dir = /work/noaa/stmp/jminsuk/stmp/jminsuk/FV3_RT/rt_182970/fv3_ccpp_control_prod Checking test 001 fv3_ccpp_control results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -71,7 +71,7 @@ Test 001 fv3_ccpp_control PASS baseline dir = /work/noaa/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_control_ccpp -working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_381876/fv3_ccpp_decomp_prod +working dir = /work/noaa/stmp/jminsuk/stmp/jminsuk/FV3_RT/rt_182970/fv3_ccpp_decomp_prod Checking test 002 fv3_ccpp_decomp results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -139,7 +139,7 @@ Test 002 fv3_ccpp_decomp PASS baseline dir = /work/noaa/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_control_ccpp -working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_381876/fv3_ccpp_2threads_prod +working dir = /work/noaa/stmp/jminsuk/stmp/jminsuk/FV3_RT/rt_182970/fv3_ccpp_2threads_prod Checking test 003 fv3_ccpp_2threads results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -207,7 +207,7 @@ Test 003 fv3_ccpp_2threads PASS baseline dir = /work/noaa/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_restart_ccpp -working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_381876/fv3_ccpp_restart_prod +working dir = /work/noaa/stmp/jminsuk/stmp/jminsuk/FV3_RT/rt_182970/fv3_ccpp_restart_prod Checking test 004 fv3_ccpp_restart results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -275,7 +275,7 @@ Test 004 fv3_ccpp_restart PASS baseline dir = /work/noaa/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_read_inc_ccpp -working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_381876/fv3_ccpp_read_inc_prod +working dir = /work/noaa/stmp/jminsuk/stmp/jminsuk/FV3_RT/rt_182970/fv3_ccpp_read_inc_prod Checking test 005 fv3_ccpp_read_inc results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -343,7 +343,7 @@ Test 005 fv3_ccpp_read_inc PASS baseline dir = /work/noaa/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_wrtGauss_netcdf_esmf_ccpp -working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_381876/fv3_ccpp_wrtGauss_netcdf_esmf_prod +working dir = /work/noaa/stmp/jminsuk/stmp/jminsuk/FV3_RT/rt_182970/fv3_ccpp_wrtGauss_netcdf_esmf_prod Checking test 006 fv3_ccpp_wrtGauss_netcdf_esmf results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -391,7 +391,7 @@ Test 006 fv3_ccpp_wrtGauss_netcdf_esmf PASS baseline dir = /work/noaa/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_wrtGauss_netcdf_ccpp -working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_381876/fv3_ccpp_wrtGauss_netcdf_prod +working dir = /work/noaa/stmp/jminsuk/stmp/jminsuk/FV3_RT/rt_182970/fv3_ccpp_wrtGauss_netcdf_prod Checking test 007 fv3_ccpp_wrtGauss_netcdf results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -439,7 +439,7 @@ Test 007 fv3_ccpp_wrtGauss_netcdf PASS baseline dir = /work/noaa/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_wrtGlatlon_netcdf_ccpp -working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_381876/fv3_ccpp_wrtGlatlon_netcdf_prod +working dir = /work/noaa/stmp/jminsuk/stmp/jminsuk/FV3_RT/rt_182970/fv3_ccpp_wrtGlatlon_netcdf_prod Checking test 008 fv3_ccpp_wrtGlatlon_netcdf results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -487,7 +487,7 @@ Test 008 fv3_ccpp_wrtGlatlon_netcdf PASS baseline dir = /work/noaa/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_wrtGauss_nemsio_ccpp -working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_381876/fv3_ccpp_wrtGauss_nemsio_prod +working dir = /work/noaa/stmp/jminsuk/stmp/jminsuk/FV3_RT/rt_182970/fv3_ccpp_wrtGauss_nemsio_prod Checking test 009 fv3_ccpp_wrtGauss_nemsio results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -535,7 +535,7 @@ Test 009 fv3_ccpp_wrtGauss_nemsio PASS baseline dir = /work/noaa/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_wrtGauss_nemsio_c192_ccpp -working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_381876/fv3_ccpp_wrtGauss_nemsio_c192_prod +working dir = /work/noaa/stmp/jminsuk/stmp/jminsuk/FV3_RT/rt_182970/fv3_ccpp_wrtGauss_nemsio_c192_prod Checking test 010 fv3_ccpp_wrtGauss_nemsio_c192 results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -583,7 +583,7 @@ Test 010 fv3_ccpp_wrtGauss_nemsio_c192 PASS baseline dir = /work/noaa/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_stochy_ccpp -working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_381876/fv3_ccpp_stochy_prod +working dir = /work/noaa/stmp/jminsuk/stmp/jminsuk/FV3_RT/rt_182970/fv3_ccpp_stochy_prod Checking test 011 fv3_ccpp_stochy results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -651,7 +651,7 @@ Test 011 fv3_ccpp_stochy PASS baseline dir = /work/noaa/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_iau_ccpp -working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_381876/fv3_ccpp_iau_prod +working dir = /work/noaa/stmp/jminsuk/stmp/jminsuk/FV3_RT/rt_182970/fv3_ccpp_iau_prod Checking test 012 fv3_ccpp_iau results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -719,7 +719,7 @@ Test 012 fv3_ccpp_iau PASS baseline dir = /work/noaa/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_ca_ccpp -working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_381876/fv3_ccpp_ca_prod +working dir = /work/noaa/stmp/jminsuk/stmp/jminsuk/FV3_RT/rt_182970/fv3_ccpp_ca_prod Checking test 013 fv3_ccpp_ca results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -787,7 +787,7 @@ Test 013 fv3_ccpp_ca PASS baseline dir = /work/noaa/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_gfdlmprad_ccpp -working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_381876/fv3_ccpp_gfdlmprad_prod +working dir = /work/noaa/stmp/jminsuk/stmp/jminsuk/FV3_RT/rt_182970/fv3_ccpp_gfdlmprad_prod Checking test 014 fv3_ccpp_gfdlmprad results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -836,7 +836,7 @@ Test 014 fv3_ccpp_gfdlmprad PASS baseline dir = /work/noaa/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_gfdlmprad_atmwav_ccpp -working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_381876/fv3_ccpp_gfdlmprad_atmwav_prod +working dir = /work/noaa/stmp/jminsuk/stmp/jminsuk/FV3_RT/rt_182970/fv3_ccpp_gfdlmprad_atmwav_prod Checking test 015 fv3_ccpp_gfdlmprad_atmwav results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -885,7 +885,7 @@ Test 015 fv3_ccpp_gfdlmprad_atmwav PASS baseline dir = /work/noaa/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_wrtGauss_nemsio_c768_ccpp -working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_381876/fv3_ccpp_wrtGauss_nemsio_c768_prod +working dir = /work/noaa/stmp/jminsuk/stmp/jminsuk/FV3_RT/rt_182970/fv3_ccpp_wrtGauss_nemsio_c768_prod Checking test 016 fv3_ccpp_wrtGauss_nemsio_c768 results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -934,7 +934,7 @@ Test 016 fv3_ccpp_wrtGauss_nemsio_c768 PASS baseline dir = /work/noaa/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_multigases_ccpp -working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_381876/fv3_ccpp_multigases_prod +working dir = /work/noaa/stmp/jminsuk/stmp/jminsuk/FV3_RT/rt_182970/fv3_ccpp_multigases_prod Checking test 017 fv3_ccpp_multigases results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1008,7 +1008,7 @@ Test 017 fv3_ccpp_multigases PASS baseline dir = /work/noaa/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_control_32bit_ccpp -working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_381876/fv3_ccpp_control_32bit_prod +working dir = /work/noaa/stmp/jminsuk/stmp/jminsuk/FV3_RT/rt_182970/fv3_ccpp_control_32bit_prod Checking test 018 fv3_ccpp_control_32bit results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1076,7 +1076,7 @@ Test 018 fv3_ccpp_control_32bit PASS baseline dir = /work/noaa/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_stretched_ccpp -working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_381876/fv3_ccpp_stretched_prod +working dir = /work/noaa/stmp/jminsuk/stmp/jminsuk/FV3_RT/rt_182970/fv3_ccpp_stretched_prod Checking test 019 fv3_ccpp_stretched results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1132,7 +1132,7 @@ Test 019 fv3_ccpp_stretched PASS baseline dir = /work/noaa/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_stretched_nest_ccpp -working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_381876/fv3_ccpp_stretched_nest_prod +working dir = /work/noaa/stmp/jminsuk/stmp/jminsuk/FV3_RT/rt_182970/fv3_ccpp_stretched_nest_prod Checking test 020 fv3_ccpp_stretched_nest results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1199,7 +1199,7 @@ Test 020 fv3_ccpp_stretched_nest PASS baseline dir = /work/noaa/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_regional_control_ccpp -working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_381876/fv3_ccpp_regional_control_prod +working dir = /work/noaa/stmp/jminsuk/stmp/jminsuk/FV3_RT/rt_182970/fv3_ccpp_regional_control_prod Checking test 021 fv3_ccpp_regional_control results .... Comparing atmos_4xdaily.nc .........OK Comparing fv3_history2d.nc .........OK @@ -1210,7 +1210,7 @@ Test 021 fv3_ccpp_regional_control PASS baseline dir = /work/noaa/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_regional_restart_ccpp -working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_381876/fv3_ccpp_regional_restart_prod +working dir = /work/noaa/stmp/jminsuk/stmp/jminsuk/FV3_RT/rt_182970/fv3_ccpp_regional_restart_prod Checking test 022 fv3_ccpp_regional_restart results .... Comparing atmos_4xdaily.nc .........OK Comparing fv3_history2d.nc .........OK @@ -1219,7 +1219,7 @@ Test 022 fv3_ccpp_regional_restart PASS baseline dir = /work/noaa/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_regional_quilt_ccpp -working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_381876/fv3_ccpp_regional_quilt_prod +working dir = /work/noaa/stmp/jminsuk/stmp/jminsuk/FV3_RT/rt_182970/fv3_ccpp_regional_quilt_prod Checking test 023 fv3_ccpp_regional_quilt results .... Comparing atmos_4xdaily.nc .........OK Comparing dynf000.nc .........OK @@ -1230,7 +1230,7 @@ Test 023 fv3_ccpp_regional_quilt PASS baseline dir = /work/noaa/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_regional_c768_ccpp -working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_381876/fv3_ccpp_regional_c768_prod +working dir = /work/noaa/stmp/jminsuk/stmp/jminsuk/FV3_RT/rt_182970/fv3_ccpp_regional_c768_prod Checking test 024 fv3_ccpp_regional_c768 results .... Comparing atmos_4xdaily.nc .........OK Comparing dynf000.nc .........OK @@ -1241,19 +1241,19 @@ Test 024 fv3_ccpp_regional_c768 PASS baseline dir = /work/noaa/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_control_ccpp -working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_381876/fv3_ccpp_control_debug_prod +working dir = /work/noaa/stmp/jminsuk/stmp/jminsuk/FV3_RT/rt_182970/fv3_ccpp_control_debug_prod Checking test 025 fv3_ccpp_control_debug results .... Test 025 fv3_ccpp_control_debug PASS baseline dir = /work/noaa/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_stretched_nest_ccpp -working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_381876/fv3_ccpp_stretched_nest_debug_prod +working dir = /work/noaa/stmp/jminsuk/stmp/jminsuk/FV3_RT/rt_182970/fv3_ccpp_stretched_nest_debug_prod Checking test 026 fv3_ccpp_stretched_nest_debug results .... Test 026 fv3_ccpp_stretched_nest_debug PASS baseline dir = /work/noaa/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_gfdlmp_ccpp -working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_381876/fv3_ccpp_gfdlmp_prod +working dir = /work/noaa/stmp/jminsuk/stmp/jminsuk/FV3_RT/rt_182970/fv3_ccpp_gfdlmp_prod Checking test 027 fv3_ccpp_gfdlmp results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1301,7 +1301,7 @@ Test 027 fv3_ccpp_gfdlmp PASS baseline dir = /work/noaa/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_gfdlmprad_gwd_ccpp -working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_381876/fv3_ccpp_gfdlmprad_gwd_prod +working dir = /work/noaa/stmp/jminsuk/stmp/jminsuk/FV3_RT/rt_182970/fv3_ccpp_gfdlmprad_gwd_prod Checking test 028 fv3_ccpp_gfdlmprad_gwd results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1349,7 +1349,7 @@ Test 028 fv3_ccpp_gfdlmprad_gwd PASS baseline dir = /work/noaa/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_gfdlmprad_noahmp_ccpp -working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_381876/fv3_ccpp_gfdlmprad_noahmp_prod +working dir = /work/noaa/stmp/jminsuk/stmp/jminsuk/FV3_RT/rt_182970/fv3_ccpp_gfdlmprad_noahmp_prod Checking test 029 fv3_ccpp_gfdlmprad_noahmp results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1397,7 +1397,7 @@ Test 029 fv3_ccpp_gfdlmprad_noahmp PASS baseline dir = /work/noaa/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_csawmg_ccpp -working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_381876/fv3_ccpp_csawmg_prod +working dir = /work/noaa/stmp/jminsuk/stmp/jminsuk/FV3_RT/rt_182970/fv3_ccpp_csawmg_prod Checking test 030 fv3_ccpp_csawmg results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1445,7 +1445,7 @@ Test 030 fv3_ccpp_csawmg PASS baseline dir = /work/noaa/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_satmedmf_ccpp -working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_381876/fv3_ccpp_satmedmf_prod +working dir = /work/noaa/stmp/jminsuk/stmp/jminsuk/FV3_RT/rt_182970/fv3_ccpp_satmedmf_prod Checking test 031 fv3_ccpp_satmedmf results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1493,7 +1493,7 @@ Test 031 fv3_ccpp_satmedmf PASS baseline dir = /work/noaa/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_gfdlmp_32bit_ccpp -working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_381876/fv3_ccpp_gfdlmp_32bit_prod +working dir = /work/noaa/stmp/jminsuk/stmp/jminsuk/FV3_RT/rt_182970/fv3_ccpp_gfdlmp_32bit_prod Checking test 032 fv3_ccpp_gfdlmp_32bit results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1541,7 +1541,7 @@ Test 032 fv3_ccpp_gfdlmp_32bit PASS baseline dir = /work/noaa/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_gfdlmprad_32bit_post_ccpp -working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_381876/fv3_ccpp_gfdlmprad_32bit_post_prod +working dir = /work/noaa/stmp/jminsuk/stmp/jminsuk/FV3_RT/rt_182970/fv3_ccpp_gfdlmprad_32bit_post_prod Checking test 033 fv3_ccpp_gfdlmprad_32bit_post results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1593,7 +1593,7 @@ Test 033 fv3_ccpp_gfdlmprad_32bit_post PASS baseline dir = /work/noaa/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_cpt_ccpp -working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_381876/fv3_ccpp_cpt_prod +working dir = /work/noaa/stmp/jminsuk/stmp/jminsuk/FV3_RT/rt_182970/fv3_ccpp_cpt_prod Checking test 034 fv3_ccpp_cpt results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1647,7 +1647,7 @@ Test 034 fv3_ccpp_cpt PASS baseline dir = /work/noaa/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_gsd_ccpp -working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_381876/fv3_ccpp_gsd_prod +working dir = /work/noaa/stmp/jminsuk/stmp/jminsuk/FV3_RT/rt_182970/fv3_ccpp_gsd_prod Checking test 035 fv3_ccpp_gsd results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1739,7 +1739,7 @@ Test 035 fv3_ccpp_gsd PASS baseline dir = /work/noaa/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_thompson_ccpp -working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_381876/fv3_ccpp_thompson_prod +working dir = /work/noaa/stmp/jminsuk/stmp/jminsuk/FV3_RT/rt_182970/fv3_ccpp_thompson_prod Checking test 036 fv3_ccpp_thompson results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1807,7 +1807,7 @@ Test 036 fv3_ccpp_thompson PASS baseline dir = /work/noaa/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_thompson_no_aero_ccpp -working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_381876/fv3_ccpp_thompson_no_aero_prod +working dir = /work/noaa/stmp/jminsuk/stmp/jminsuk/FV3_RT/rt_182970/fv3_ccpp_thompson_no_aero_prod Checking test 037 fv3_ccpp_thompson_no_aero results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1875,7 +1875,7 @@ Test 037 fv3_ccpp_thompson_no_aero PASS baseline dir = /work/noaa/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_rrfs_v1beta_ccpp -working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_381876/fv3_ccpp_rrfs_v1beta_prod +working dir = /work/noaa/stmp/jminsuk/stmp/jminsuk/FV3_RT/rt_182970/fv3_ccpp_rrfs_v1beta_prod Checking test 038 fv3_ccpp_rrfs_v1beta results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1967,7 +1967,7 @@ Test 038 fv3_ccpp_rrfs_v1beta PASS baseline dir = /work/noaa/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_gfs_v15p2_ccpp -working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_381876/fv3_ccpp_gfs_v15p2_prod +working dir = /work/noaa/stmp/jminsuk/stmp/jminsuk/FV3_RT/rt_182970/fv3_ccpp_gfs_v15p2_prod Checking test 039 fv3_ccpp_gfs_v15p2 results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2035,7 +2035,7 @@ Test 039 fv3_ccpp_gfs_v15p2 PASS baseline dir = /work/noaa/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_gfs_v16beta_ccpp -working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_381876/fv3_ccpp_gfs_v16beta_prod +working dir = /work/noaa/stmp/jminsuk/stmp/jminsuk/FV3_RT/rt_182970/fv3_ccpp_gfs_v16beta_prod Checking test 040 fv3_ccpp_gfs_v16beta results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2103,7 +2103,7 @@ Test 040 fv3_ccpp_gfs_v16beta PASS baseline dir = /work/noaa/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_gfs_v15p2_RRTMGP_ccpp -working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_381876/fv3_ccpp_gfs_v15p2_RRTMGP_prod +working dir = /work/noaa/stmp/jminsuk/stmp/jminsuk/FV3_RT/rt_182970/fv3_ccpp_gfs_v15p2_RRTMGP_prod Checking test 041 fv3_ccpp_gfs_v15p2_RRTMGP results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2171,7 +2171,7 @@ Test 041 fv3_ccpp_gfs_v15p2_RRTMGP PASS baseline dir = /work/noaa/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_gfs_v16beta_RRTMGP_ccpp -working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_381876/fv3_ccpp_gfs_v16beta_RRTMGP_prod +working dir = /work/noaa/stmp/jminsuk/stmp/jminsuk/FV3_RT/rt_182970/fv3_ccpp_gfs_v16beta_RRTMGP_prod Checking test 042 fv3_ccpp_gfs_v16beta_RRTMGP results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2239,7 +2239,7 @@ Test 042 fv3_ccpp_gfs_v16beta_RRTMGP PASS baseline dir = /work/noaa/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_gfsv16_csawmg_ccpp -working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_381876/fv3_ccpp_gfsv16_csawmg_prod +working dir = /work/noaa/stmp/jminsuk/stmp/jminsuk/FV3_RT/rt_182970/fv3_ccpp_gfsv16_csawmg_prod Checking test 043 fv3_ccpp_gfsv16_csawmg results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2287,7 +2287,7 @@ Test 043 fv3_ccpp_gfsv16_csawmg PASS baseline dir = /work/noaa/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_gfsv16_csawmgt_ccpp -working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_381876/fv3_ccpp_gfsv16_csawmgt_prod +working dir = /work/noaa/stmp/jminsuk/stmp/jminsuk/FV3_RT/rt_182970/fv3_ccpp_gfsv16_csawmgt_prod Checking test 044 fv3_ccpp_gfsv16_csawmgt results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2335,7 +2335,7 @@ Test 044 fv3_ccpp_gfsv16_csawmgt PASS baseline dir = /work/noaa/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_gocart_clm_ccpp -working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_381876/fv3_ccpp_gocart_clm_prod +working dir = /work/noaa/stmp/jminsuk/stmp/jminsuk/FV3_RT/rt_182970/fv3_ccpp_gocart_clm_prod Checking test 045 fv3_ccpp_gocart_clm results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2383,7 +2383,7 @@ Test 045 fv3_ccpp_gocart_clm PASS baseline dir = /work/noaa/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_gfs_v16beta_flake_ccpp -working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_381876/fv3_ccpp_gfs_v16beta_flake_prod +working dir = /work/noaa/stmp/jminsuk/stmp/jminsuk/FV3_RT/rt_182970/fv3_ccpp_gfs_v16beta_flake_prod Checking test 046 fv3_ccpp_gfs_v16beta_flake results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2451,7 +2451,7 @@ Test 046 fv3_ccpp_gfs_v16beta_flake PASS baseline dir = /work/noaa/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_gfs_v15p2_debug_ccpp -working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_381876/fv3_ccpp_gfs_v15p2_debug_prod +working dir = /work/noaa/stmp/jminsuk/stmp/jminsuk/FV3_RT/rt_182970/fv3_ccpp_gfs_v15p2_debug_prod Checking test 047 fv3_ccpp_gfs_v15p2_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2519,7 +2519,7 @@ Test 047 fv3_ccpp_gfs_v15p2_debug PASS baseline dir = /work/noaa/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_gfs_v16beta_debug_ccpp -working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_381876/fv3_ccpp_gfs_v16beta_debug_prod +working dir = /work/noaa/stmp/jminsuk/stmp/jminsuk/FV3_RT/rt_182970/fv3_ccpp_gfs_v16beta_debug_prod Checking test 048 fv3_ccpp_gfs_v16beta_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2587,7 +2587,7 @@ Test 048 fv3_ccpp_gfs_v16beta_debug PASS baseline dir = /work/noaa/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_gfs_v15p2_RRTMGP_debug_ccpp -working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_381876/fv3_ccpp_gfs_v15p2_RRTMGP_debug_prod +working dir = /work/noaa/stmp/jminsuk/stmp/jminsuk/FV3_RT/rt_182970/fv3_ccpp_gfs_v15p2_RRTMGP_debug_prod Checking test 049 fv3_ccpp_gfs_v15p2_RRTMGP_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2655,7 +2655,7 @@ Test 049 fv3_ccpp_gfs_v15p2_RRTMGP_debug PASS baseline dir = /work/noaa/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_gfs_v16beta_RRTMGP_debug_ccpp -working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_381876/fv3_ccpp_gfs_v16beta_RRTMGP_debug_prod +working dir = /work/noaa/stmp/jminsuk/stmp/jminsuk/FV3_RT/rt_182970/fv3_ccpp_gfs_v16beta_RRTMGP_debug_prod Checking test 050 fv3_ccpp_gfs_v16beta_RRTMGP_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2723,7 +2723,7 @@ Test 050 fv3_ccpp_gfs_v16beta_RRTMGP_debug PASS baseline dir = /work/noaa/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_gsd_debug_ccpp -working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_381876/fv3_ccpp_gsd_debug_prod +working dir = /work/noaa/stmp/jminsuk/stmp/jminsuk/FV3_RT/rt_182970/fv3_ccpp_gsd_debug_prod Checking test 051 fv3_ccpp_gsd_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2791,7 +2791,7 @@ Test 051 fv3_ccpp_gsd_debug PASS baseline dir = /work/noaa/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_gsd_diag3d_debug_ccpp -working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_381876/fv3_ccpp_gsd_diag3d_debug_prod +working dir = /work/noaa/stmp/jminsuk/stmp/jminsuk/FV3_RT/rt_182970/fv3_ccpp_gsd_diag3d_debug_prod Checking test 052 fv3_ccpp_gsd_diag3d_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2859,7 +2859,7 @@ Test 052 fv3_ccpp_gsd_diag3d_debug PASS baseline dir = /work/noaa/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_thompson_debug_ccpp -working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_381876/fv3_ccpp_thompson_debug_prod +working dir = /work/noaa/stmp/jminsuk/stmp/jminsuk/FV3_RT/rt_182970/fv3_ccpp_thompson_debug_prod Checking test 053 fv3_ccpp_thompson_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2927,7 +2927,7 @@ Test 053 fv3_ccpp_thompson_debug PASS baseline dir = /work/noaa/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_thompson_no_aero_debug_ccpp -working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_381876/fv3_ccpp_thompson_no_aero_debug_prod +working dir = /work/noaa/stmp/jminsuk/stmp/jminsuk/FV3_RT/rt_182970/fv3_ccpp_thompson_no_aero_debug_prod Checking test 054 fv3_ccpp_thompson_no_aero_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2995,7 +2995,7 @@ Test 054 fv3_ccpp_thompson_no_aero_debug PASS baseline dir = /work/noaa/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/INTEL/fv3_rrfs_v1beta_debug_ccpp -working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_381876/fv3_ccpp_rrfs_v1beta_debug_prod +working dir = /work/noaa/stmp/jminsuk/stmp/jminsuk/FV3_RT/rt_182970/fv3_ccpp_rrfs_v1beta_debug_prod Checking test 055 fv3_ccpp_rrfs_v1beta_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -3063,5 +3063,5 @@ Test 055 fv3_ccpp_rrfs_v1beta_debug PASS REGRESSION TEST WAS SUCCESSFUL -Tue Sep 1 15:43:27 CDT 2020 -Elapsed time: 00h:49m:21s. Have a nice day! +Tue Sep 8 16:54:40 CDT 2020 +Elapsed time: 01h:41m:36s. Have a nice day! diff --git a/tests/RegressionTests_wcoss_dell_p3.log b/tests/RegressionTests_wcoss_dell_p3.log index e667499b23..c7175b3e5d 100644 --- a/tests/RegressionTests_wcoss_dell_p3.log +++ b/tests/RegressionTests_wcoss_dell_p3.log @@ -1,9 +1,9 @@ -Tue Sep 1 17:52:55 UTC 2020 +Tue Sep 8 20:04:31 UTC 2020 Start Regression test baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/fv3_control_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_169279/fv3_ccpp_control_prod +working dir = /gpfs/dell2/ptmp/Minsuk.Ji/FV3_RT/rt_28520/fv3_ccpp_control_prod Checking test 001 fv3_ccpp_control results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -71,7 +71,7 @@ Test 001 fv3_ccpp_control PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/fv3_control_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_169279/fv3_ccpp_decomp_prod +working dir = /gpfs/dell2/ptmp/Minsuk.Ji/FV3_RT/rt_28520/fv3_ccpp_decomp_prod Checking test 002 fv3_ccpp_decomp results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -139,7 +139,7 @@ Test 002 fv3_ccpp_decomp PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/fv3_control_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_169279/fv3_ccpp_2threads_prod +working dir = /gpfs/dell2/ptmp/Minsuk.Ji/FV3_RT/rt_28520/fv3_ccpp_2threads_prod Checking test 003 fv3_ccpp_2threads results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -207,7 +207,7 @@ Test 003 fv3_ccpp_2threads PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/fv3_restart_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_169279/fv3_ccpp_restart_prod +working dir = /gpfs/dell2/ptmp/Minsuk.Ji/FV3_RT/rt_28520/fv3_ccpp_restart_prod Checking test 004 fv3_ccpp_restart results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -275,7 +275,7 @@ Test 004 fv3_ccpp_restart PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/fv3_read_inc_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_169279/fv3_ccpp_read_inc_prod +working dir = /gpfs/dell2/ptmp/Minsuk.Ji/FV3_RT/rt_28520/fv3_ccpp_read_inc_prod Checking test 005 fv3_ccpp_read_inc results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -343,7 +343,7 @@ Test 005 fv3_ccpp_read_inc PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/fv3_wrtGauss_netcdf_esmf_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_169279/fv3_ccpp_wrtGauss_netcdf_esmf_prod +working dir = /gpfs/dell2/ptmp/Minsuk.Ji/FV3_RT/rt_28520/fv3_ccpp_wrtGauss_netcdf_esmf_prod Checking test 006 fv3_ccpp_wrtGauss_netcdf_esmf results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -391,7 +391,7 @@ Test 006 fv3_ccpp_wrtGauss_netcdf_esmf PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/fv3_wrtGauss_netcdf_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_169279/fv3_ccpp_wrtGauss_netcdf_prod +working dir = /gpfs/dell2/ptmp/Minsuk.Ji/FV3_RT/rt_28520/fv3_ccpp_wrtGauss_netcdf_prod Checking test 007 fv3_ccpp_wrtGauss_netcdf results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -439,7 +439,7 @@ Test 007 fv3_ccpp_wrtGauss_netcdf PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/fv3_wrtGlatlon_netcdf_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_169279/fv3_ccpp_wrtGlatlon_netcdf_prod +working dir = /gpfs/dell2/ptmp/Minsuk.Ji/FV3_RT/rt_28520/fv3_ccpp_wrtGlatlon_netcdf_prod Checking test 008 fv3_ccpp_wrtGlatlon_netcdf results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -487,7 +487,7 @@ Test 008 fv3_ccpp_wrtGlatlon_netcdf PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/fv3_wrtGauss_nemsio_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_169279/fv3_ccpp_wrtGauss_nemsio_prod +working dir = /gpfs/dell2/ptmp/Minsuk.Ji/FV3_RT/rt_28520/fv3_ccpp_wrtGauss_nemsio_prod Checking test 009 fv3_ccpp_wrtGauss_nemsio results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -535,7 +535,7 @@ Test 009 fv3_ccpp_wrtGauss_nemsio PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/fv3_wrtGauss_nemsio_c192_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_169279/fv3_ccpp_wrtGauss_nemsio_c192_prod +working dir = /gpfs/dell2/ptmp/Minsuk.Ji/FV3_RT/rt_28520/fv3_ccpp_wrtGauss_nemsio_c192_prod Checking test 010 fv3_ccpp_wrtGauss_nemsio_c192 results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -583,7 +583,7 @@ Test 010 fv3_ccpp_wrtGauss_nemsio_c192 PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/fv3_stochy_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_169279/fv3_ccpp_stochy_prod +working dir = /gpfs/dell2/ptmp/Minsuk.Ji/FV3_RT/rt_28520/fv3_ccpp_stochy_prod Checking test 011 fv3_ccpp_stochy results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -651,7 +651,7 @@ Test 011 fv3_ccpp_stochy PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/fv3_iau_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_169279/fv3_ccpp_iau_prod +working dir = /gpfs/dell2/ptmp/Minsuk.Ji/FV3_RT/rt_28520/fv3_ccpp_iau_prod Checking test 012 fv3_ccpp_iau results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -719,7 +719,7 @@ Test 012 fv3_ccpp_iau PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/fv3_lheatstrg_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_169279/fv3_ccpp_lheatstrg_prod +working dir = /gpfs/dell2/ptmp/Minsuk.Ji/FV3_RT/rt_28520/fv3_ccpp_lheatstrg_prod Checking test 013 fv3_ccpp_lheatstrg results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -767,7 +767,7 @@ Test 013 fv3_ccpp_lheatstrg PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/fv3_gfdlmprad_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_169279/fv3_ccpp_gfdlmprad_prod +working dir = /gpfs/dell2/ptmp/Minsuk.Ji/FV3_RT/rt_28520/fv3_ccpp_gfdlmprad_prod Checking test 014 fv3_ccpp_gfdlmprad results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -816,7 +816,7 @@ Test 014 fv3_ccpp_gfdlmprad PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/fv3_gfdlmprad_atmwav_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_169279/fv3_ccpp_gfdlmprad_atmwav_prod +working dir = /gpfs/dell2/ptmp/Minsuk.Ji/FV3_RT/rt_28520/fv3_ccpp_gfdlmprad_atmwav_prod Checking test 015 fv3_ccpp_gfdlmprad_atmwav results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -865,7 +865,7 @@ Test 015 fv3_ccpp_gfdlmprad_atmwav PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/fv3_multigases_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_169279/fv3_ccpp_multigases_prod +working dir = /gpfs/dell2/ptmp/Minsuk.Ji/FV3_RT/rt_28520/fv3_ccpp_multigases_prod Checking test 016 fv3_ccpp_multigases results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -939,7 +939,7 @@ Test 016 fv3_ccpp_multigases PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/fv3_control_32bit_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_169279/fv3_ccpp_control_32bit_prod +working dir = /gpfs/dell2/ptmp/Minsuk.Ji/FV3_RT/rt_28520/fv3_ccpp_control_32bit_prod Checking test 017 fv3_ccpp_control_32bit results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1007,7 +1007,7 @@ Test 017 fv3_ccpp_control_32bit PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/fv3_stretched_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_169279/fv3_ccpp_stretched_prod +working dir = /gpfs/dell2/ptmp/Minsuk.Ji/FV3_RT/rt_28520/fv3_ccpp_stretched_prod Checking test 018 fv3_ccpp_stretched results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1063,7 +1063,7 @@ Test 018 fv3_ccpp_stretched PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/fv3_stretched_nest_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_169279/fv3_ccpp_stretched_nest_prod +working dir = /gpfs/dell2/ptmp/Minsuk.Ji/FV3_RT/rt_28520/fv3_ccpp_stretched_nest_prod Checking test 019 fv3_ccpp_stretched_nest results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1130,7 +1130,7 @@ Test 019 fv3_ccpp_stretched_nest PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/fv3_regional_control_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_169279/fv3_ccpp_regional_control_prod +working dir = /gpfs/dell2/ptmp/Minsuk.Ji/FV3_RT/rt_28520/fv3_ccpp_regional_control_prod Checking test 020 fv3_ccpp_regional_control results .... Comparing atmos_4xdaily.nc .........OK Comparing fv3_history2d.nc .........OK @@ -1141,7 +1141,7 @@ Test 020 fv3_ccpp_regional_control PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/fv3_regional_restart_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_169279/fv3_ccpp_regional_restart_prod +working dir = /gpfs/dell2/ptmp/Minsuk.Ji/FV3_RT/rt_28520/fv3_ccpp_regional_restart_prod Checking test 021 fv3_ccpp_regional_restart results .... Comparing atmos_4xdaily.nc .........OK Comparing fv3_history2d.nc .........OK @@ -1150,7 +1150,7 @@ Test 021 fv3_ccpp_regional_restart PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/fv3_regional_quilt_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_169279/fv3_ccpp_regional_quilt_prod +working dir = /gpfs/dell2/ptmp/Minsuk.Ji/FV3_RT/rt_28520/fv3_ccpp_regional_quilt_prod Checking test 022 fv3_ccpp_regional_quilt results .... Comparing atmos_4xdaily.nc .........OK Comparing dynf000.nc .........OK @@ -1161,7 +1161,7 @@ Test 022 fv3_ccpp_regional_quilt PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/fv3_regional_c768_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_169279/fv3_ccpp_regional_c768_prod +working dir = /gpfs/dell2/ptmp/Minsuk.Ji/FV3_RT/rt_28520/fv3_ccpp_regional_c768_prod Checking test 023 fv3_ccpp_regional_c768 results .... Comparing atmos_4xdaily.nc .........OK Comparing dynf000.nc .........OK @@ -1172,19 +1172,19 @@ Test 023 fv3_ccpp_regional_c768 PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/fv3_control_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_169279/fv3_ccpp_control_debug_prod +working dir = /gpfs/dell2/ptmp/Minsuk.Ji/FV3_RT/rt_28520/fv3_ccpp_control_debug_prod Checking test 024 fv3_ccpp_control_debug results .... Test 024 fv3_ccpp_control_debug PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/fv3_stretched_nest_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_169279/fv3_ccpp_stretched_nest_debug_prod +working dir = /gpfs/dell2/ptmp/Minsuk.Ji/FV3_RT/rt_28520/fv3_ccpp_stretched_nest_debug_prod Checking test 025 fv3_ccpp_stretched_nest_debug results .... Test 025 fv3_ccpp_stretched_nest_debug PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/fv3_gfdlmp_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_169279/fv3_ccpp_gfdlmp_prod +working dir = /gpfs/dell2/ptmp/Minsuk.Ji/FV3_RT/rt_28520/fv3_ccpp_gfdlmp_prod Checking test 026 fv3_ccpp_gfdlmp results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1232,7 +1232,7 @@ Test 026 fv3_ccpp_gfdlmp PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/fv3_gfdlmprad_gwd_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_169279/fv3_ccpp_gfdlmprad_gwd_prod +working dir = /gpfs/dell2/ptmp/Minsuk.Ji/FV3_RT/rt_28520/fv3_ccpp_gfdlmprad_gwd_prod Checking test 027 fv3_ccpp_gfdlmprad_gwd results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1280,7 +1280,7 @@ Test 027 fv3_ccpp_gfdlmprad_gwd PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/fv3_gfdlmprad_noahmp_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_169279/fv3_ccpp_gfdlmprad_noahmp_prod +working dir = /gpfs/dell2/ptmp/Minsuk.Ji/FV3_RT/rt_28520/fv3_ccpp_gfdlmprad_noahmp_prod Checking test 028 fv3_ccpp_gfdlmprad_noahmp results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1328,7 +1328,7 @@ Test 028 fv3_ccpp_gfdlmprad_noahmp PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/fv3_gfdlmp_hwrfsas_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_169279/fv3_ccpp_gfdlmp_hwrfsas_prod +working dir = /gpfs/dell2/ptmp/Minsuk.Ji/FV3_RT/rt_28520/fv3_ccpp_gfdlmp_hwrfsas_prod Checking test 029 fv3_ccpp_gfdlmp_hwrfsas results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1376,7 +1376,7 @@ Test 029 fv3_ccpp_gfdlmp_hwrfsas PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/fv3_csawmg_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_169279/fv3_ccpp_csawmg_prod +working dir = /gpfs/dell2/ptmp/Minsuk.Ji/FV3_RT/rt_28520/fv3_ccpp_csawmg_prod Checking test 030 fv3_ccpp_csawmg results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1424,7 +1424,7 @@ Test 030 fv3_ccpp_csawmg PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/fv3_satmedmf_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_169279/fv3_ccpp_satmedmf_prod +working dir = /gpfs/dell2/ptmp/Minsuk.Ji/FV3_RT/rt_28520/fv3_ccpp_satmedmf_prod Checking test 031 fv3_ccpp_satmedmf results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1472,7 +1472,7 @@ Test 031 fv3_ccpp_satmedmf PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/fv3_satmedmfq_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_169279/fv3_ccpp_satmedmfq_prod +working dir = /gpfs/dell2/ptmp/Minsuk.Ji/FV3_RT/rt_28520/fv3_ccpp_satmedmfq_prod Checking test 032 fv3_ccpp_satmedmfq results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1520,7 +1520,7 @@ Test 032 fv3_ccpp_satmedmfq PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/fv3_gfdlmp_32bit_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_169279/fv3_ccpp_gfdlmp_32bit_prod +working dir = /gpfs/dell2/ptmp/Minsuk.Ji/FV3_RT/rt_28520/fv3_ccpp_gfdlmp_32bit_prod Checking test 033 fv3_ccpp_gfdlmp_32bit results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1568,7 +1568,7 @@ Test 033 fv3_ccpp_gfdlmp_32bit PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/fv3_gfdlmprad_32bit_post_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_169279/fv3_ccpp_gfdlmprad_32bit_post_prod +working dir = /gpfs/dell2/ptmp/Minsuk.Ji/FV3_RT/rt_28520/fv3_ccpp_gfdlmprad_32bit_post_prod Checking test 034 fv3_ccpp_gfdlmprad_32bit_post results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1620,7 +1620,7 @@ Test 034 fv3_ccpp_gfdlmprad_32bit_post PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/fv3_cpt_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_169279/fv3_ccpp_cpt_prod +working dir = /gpfs/dell2/ptmp/Minsuk.Ji/FV3_RT/rt_28520/fv3_ccpp_cpt_prod Checking test 035 fv3_ccpp_cpt results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1674,7 +1674,7 @@ Test 035 fv3_ccpp_cpt PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/fv3_gsd_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_169279/fv3_ccpp_gsd_prod +working dir = /gpfs/dell2/ptmp/Minsuk.Ji/FV3_RT/rt_28520/fv3_ccpp_gsd_prod Checking test 036 fv3_ccpp_gsd results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1766,7 +1766,7 @@ Test 036 fv3_ccpp_gsd PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/fv3_thompson_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_169279/fv3_ccpp_thompson_prod +working dir = /gpfs/dell2/ptmp/Minsuk.Ji/FV3_RT/rt_28520/fv3_ccpp_thompson_prod Checking test 037 fv3_ccpp_thompson results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1834,7 +1834,7 @@ Test 037 fv3_ccpp_thompson PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/fv3_thompson_no_aero_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_169279/fv3_ccpp_thompson_no_aero_prod +working dir = /gpfs/dell2/ptmp/Minsuk.Ji/FV3_RT/rt_28520/fv3_ccpp_thompson_no_aero_prod Checking test 038 fv3_ccpp_thompson_no_aero results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1902,7 +1902,7 @@ Test 038 fv3_ccpp_thompson_no_aero PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/fv3_rrfs_v1beta_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_169279/fv3_ccpp_rrfs_v1beta_prod +working dir = /gpfs/dell2/ptmp/Minsuk.Ji/FV3_RT/rt_28520/fv3_ccpp_rrfs_v1beta_prod Checking test 039 fv3_ccpp_rrfs_v1beta results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1994,7 +1994,7 @@ Test 039 fv3_ccpp_rrfs_v1beta PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/fv3_gfsv16_csawmg_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_169279/fv3_ccpp_gfsv16_csawmg_prod +working dir = /gpfs/dell2/ptmp/Minsuk.Ji/FV3_RT/rt_28520/fv3_ccpp_gfsv16_csawmg_prod Checking test 040 fv3_ccpp_gfsv16_csawmg results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2042,7 +2042,7 @@ Test 040 fv3_ccpp_gfsv16_csawmg PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/fv3_gfsv16_csawmgt_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_169279/fv3_ccpp_gfsv16_csawmgt_prod +working dir = /gpfs/dell2/ptmp/Minsuk.Ji/FV3_RT/rt_28520/fv3_ccpp_gfsv16_csawmgt_prod Checking test 041 fv3_ccpp_gfsv16_csawmgt results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2090,7 +2090,7 @@ Test 041 fv3_ccpp_gfsv16_csawmgt PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/fv3_gocart_clm_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_169279/fv3_ccpp_gocart_clm_prod +working dir = /gpfs/dell2/ptmp/Minsuk.Ji/FV3_RT/rt_28520/fv3_ccpp_gocart_clm_prod Checking test 042 fv3_ccpp_gocart_clm results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2138,7 +2138,7 @@ Test 042 fv3_ccpp_gocart_clm PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/fv3_gfs_v16beta_flake_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_169279/fv3_ccpp_gfs_v16beta_flake_prod +working dir = /gpfs/dell2/ptmp/Minsuk.Ji/FV3_RT/rt_28520/fv3_ccpp_gfs_v16beta_flake_prod Checking test 043 fv3_ccpp_gfs_v16beta_flake results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2206,7 +2206,7 @@ Test 043 fv3_ccpp_gfs_v16beta_flake PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/fv3_gsd_debug_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_169279/fv3_ccpp_gsd_debug_prod +working dir = /gpfs/dell2/ptmp/Minsuk.Ji/FV3_RT/rt_28520/fv3_ccpp_gsd_debug_prod Checking test 044 fv3_ccpp_gsd_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2274,7 +2274,7 @@ Test 044 fv3_ccpp_gsd_debug PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/fv3_gsd_diag3d_debug_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_169279/fv3_ccpp_gsd_diag3d_debug_prod +working dir = /gpfs/dell2/ptmp/Minsuk.Ji/FV3_RT/rt_28520/fv3_ccpp_gsd_diag3d_debug_prod Checking test 045 fv3_ccpp_gsd_diag3d_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2342,7 +2342,7 @@ Test 045 fv3_ccpp_gsd_diag3d_debug PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/fv3_thompson_debug_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_169279/fv3_ccpp_thompson_debug_prod +working dir = /gpfs/dell2/ptmp/Minsuk.Ji/FV3_RT/rt_28520/fv3_ccpp_thompson_debug_prod Checking test 046 fv3_ccpp_thompson_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2410,7 +2410,7 @@ Test 046 fv3_ccpp_thompson_debug PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/fv3_thompson_no_aero_debug_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_169279/fv3_ccpp_thompson_no_aero_debug_prod +working dir = /gpfs/dell2/ptmp/Minsuk.Ji/FV3_RT/rt_28520/fv3_ccpp_thompson_no_aero_debug_prod Checking test 047 fv3_ccpp_thompson_no_aero_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2478,7 +2478,7 @@ Test 047 fv3_ccpp_thompson_no_aero_debug PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200825/fv3_rrfs_v1beta_debug_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_169279/fv3_ccpp_rrfs_v1beta_debug_prod +working dir = /gpfs/dell2/ptmp/Minsuk.Ji/FV3_RT/rt_28520/fv3_ccpp_rrfs_v1beta_debug_prod Checking test 048 fv3_ccpp_rrfs_v1beta_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2546,5 +2546,5 @@ Test 048 fv3_ccpp_rrfs_v1beta_debug PASS REGRESSION TEST WAS SUCCESSFUL -Tue Sep 1 18:58:20 UTC 2020 -Elapsed time: 01h:05m:31s. Have a nice day! +Tue Sep 8 21:05:41 UTC 2020 +Elapsed time: 01h:01m:14s. Have a nice day! diff --git a/tests/UnitTests_hera.intel.log b/tests/UnitTests_hera.intel.log index 81412d90dc..65a1bd9615 100644 --- a/tests/UnitTests_hera.intel.log +++ b/tests/UnitTests_hera.intel.log @@ -1,157 +1,147 @@ -baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200512/INTEL/fv3_control_32bit -working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_UT/ut_72447/fv3_ccpp_control_32bit_base -Checking test 32bit_base fv3_ccpp_control results .... - mkdir -p /scratch1/NCEPDEV/stmp4/Minsuk.Ji/FV3_UT/UNIT_TEST/fv3_control_32bit/RESTART - Moving atmos_4xdaily.tile1.nc ..... Moving atmos_4xdaily.tile2.nc ..... Moving atmos_4xdaily.tile3.nc ..... Moving atmos_4xdaily.tile4.nc ..... Moving atmos_4xdaily.tile5.nc ..... Moving atmos_4xdaily.tile6.nc ..... Moving phyf000.tile1.nc ..... Moving phyf000.tile2.nc ..... Moving phyf000.tile3.nc ..... Moving phyf000.tile4.nc ..... Moving phyf000.tile5.nc ..... Moving phyf000.tile6.nc ..... Moving phyf024.tile1.nc ..... Moving phyf024.tile2.nc ..... Moving phyf024.tile3.nc ..... Moving phyf024.tile4.nc ..... Moving phyf024.tile5.nc ..... Moving phyf024.tile6.nc ..... Moving dynf000.tile1.nc ..... Moving dynf000.tile2.nc ..... Moving dynf000.tile3.nc ..... Moving dynf000.tile4.nc ..... Moving dynf000.tile5.nc ..... Moving dynf000.tile6.nc ..... Moving dynf024.tile1.nc ..... Moving dynf024.tile2.nc ..... Moving dynf024.tile3.nc ..... Moving dynf024.tile4.nc ..... Moving dynf024.tile5.nc ..... Moving dynf024.tile6.nc ..... Moving RESTART/coupler.res ..... Moving RESTART/fv_core.res.nc ..... Moving RESTART/fv_core.res.tile1.nc ..... Moving RESTART/fv_core.res.tile2.nc ..... Moving RESTART/fv_core.res.tile3.nc ..... Moving RESTART/fv_core.res.tile4.nc ..... Moving RESTART/fv_core.res.tile5.nc ..... Moving RESTART/fv_core.res.tile6.nc ..... Moving RESTART/fv_srf_wnd.res.tile1.nc ..... Moving RESTART/fv_srf_wnd.res.tile2.nc ..... Moving RESTART/fv_srf_wnd.res.tile3.nc ..... Moving RESTART/fv_srf_wnd.res.tile4.nc ..... Moving RESTART/fv_srf_wnd.res.tile5.nc ..... Moving RESTART/fv_srf_wnd.res.tile6.nc ..... Moving RESTART/fv_tracer.res.tile1.nc ..... Moving RESTART/fv_tracer.res.tile2.nc ..... Moving RESTART/fv_tracer.res.tile3.nc ..... Moving RESTART/fv_tracer.res.tile4.nc ..... Moving RESTART/fv_tracer.res.tile5.nc ..... Moving RESTART/fv_tracer.res.tile6.nc ..... Moving RESTART/phy_data.tile1.nc ..... Moving RESTART/phy_data.tile2.nc ..... Moving RESTART/phy_data.tile3.nc ..... Moving RESTART/phy_data.tile4.nc ..... Moving RESTART/phy_data.tile5.nc ..... Moving RESTART/phy_data.tile6.nc ..... Moving RESTART/sfc_data.tile1.nc ..... Moving RESTART/sfc_data.tile2.nc ..... Moving RESTART/sfc_data.tile3.nc ..... Moving RESTART/sfc_data.tile4.nc ..... Moving RESTART/sfc_data.tile5.nc ..... Moving RESTART/sfc_data.tile6.nc .....Test 32bit_base fv3_ccpp_control PASS +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_control_bit_base +working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_UT/ut_269251/fv3_ccpp_control_bit_base +Checking test bit_base fv3_ccpp_control results .... +Moving baseline bit_base fv3_ccpp_control files .... + mkdir -p /scratch1/NCEPDEV/stmp4/Minsuk.Ji/FV3_UT/UNIT_TEST/fv3_control_bit_base/RESTART + Moving atmos_4xdaily.tile1.nc .........OK + Moving atmos_4xdaily.tile2.nc .........OK + Moving atmos_4xdaily.tile3.nc .........OK + Moving atmos_4xdaily.tile4.nc .........OK + Moving atmos_4xdaily.tile5.nc .........OK + Moving atmos_4xdaily.tile6.nc .........OK + Moving phyf000.tile1.nc .........OK + Moving phyf000.tile2.nc .........OK + Moving phyf000.tile3.nc .........OK + Moving phyf000.tile4.nc .........OK + Moving phyf000.tile5.nc .........OK + Moving phyf000.tile6.nc .........OK + Moving phyf024.tile1.nc .........OK + Moving phyf024.tile2.nc .........OK + Moving phyf024.tile3.nc .........OK + Moving phyf024.tile4.nc .........OK + Moving phyf024.tile5.nc .........OK + Moving phyf024.tile6.nc .........OK + Moving dynf000.tile1.nc .........OK + Moving dynf000.tile2.nc .........OK + Moving dynf000.tile3.nc .........OK + Moving dynf000.tile4.nc .........OK + Moving dynf000.tile5.nc .........OK + Moving dynf000.tile6.nc .........OK + Moving dynf024.tile1.nc .........OK + Moving dynf024.tile2.nc .........OK + Moving dynf024.tile3.nc .........OK + Moving dynf024.tile4.nc .........OK + Moving dynf024.tile5.nc .........OK + Moving dynf024.tile6.nc .........OK + Moving RESTART/coupler.res .........OK + Moving RESTART/fv_core.res.nc .........OK + Moving RESTART/fv_core.res.tile1.nc .........OK + Moving RESTART/fv_core.res.tile2.nc .........OK + Moving RESTART/fv_core.res.tile3.nc .........OK + Moving RESTART/fv_core.res.tile4.nc .........OK + Moving RESTART/fv_core.res.tile5.nc .........OK + Moving RESTART/fv_core.res.tile6.nc .........OK + Moving RESTART/fv_srf_wnd.res.tile1.nc .........OK + Moving RESTART/fv_srf_wnd.res.tile2.nc .........OK + Moving RESTART/fv_srf_wnd.res.tile3.nc .........OK + Moving RESTART/fv_srf_wnd.res.tile4.nc .........OK + Moving RESTART/fv_srf_wnd.res.tile5.nc .........OK + Moving RESTART/fv_srf_wnd.res.tile6.nc .........OK + Moving RESTART/fv_tracer.res.tile1.nc .........OK + Moving RESTART/fv_tracer.res.tile2.nc .........OK + Moving RESTART/fv_tracer.res.tile3.nc .........OK + Moving RESTART/fv_tracer.res.tile4.nc .........OK + Moving RESTART/fv_tracer.res.tile5.nc .........OK + Moving RESTART/fv_tracer.res.tile6.nc .........OK + Moving RESTART/phy_data.tile1.nc .........OK + Moving RESTART/phy_data.tile2.nc .........OK + Moving RESTART/phy_data.tile3.nc .........OK + Moving RESTART/phy_data.tile4.nc .........OK + Moving RESTART/phy_data.tile5.nc .........OK + Moving RESTART/phy_data.tile6.nc .........OK + Moving RESTART/sfc_data.tile1.nc .........OK + Moving RESTART/sfc_data.tile2.nc .........OK + Moving RESTART/sfc_data.tile3.nc .........OK + Moving RESTART/sfc_data.tile4.nc .........OK + Moving RESTART/sfc_data.tile5.nc .........OK + Moving RESTART/sfc_data.tile6.nc .........OK +Test bit_base fv3_ccpp_control PASS -baseline dir = /scratch1/NCEPDEV/stmp4/Minsuk.Ji/FV3_UT/UNIT_TEST/fv3_control_32bit -working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_UT/ut_72447/fv3_ccpp_control_32bit -Checking test 32bit fv3_ccpp_control results .... - Comparing atmos_4xdaily.tile1.nc .........OK - Comparing atmos_4xdaily.tile2.nc .........OK - Comparing atmos_4xdaily.tile3.nc .........OK - Comparing atmos_4xdaily.tile4.nc .........OK - Comparing atmos_4xdaily.tile5.nc .........OK - Comparing atmos_4xdaily.tile6.nc .........OK - Comparing phyf000.tile1.nc .........OK - Comparing phyf000.tile2.nc .........OK - Comparing phyf000.tile3.nc .........OK - Comparing phyf000.tile4.nc .........OK - Comparing phyf000.tile5.nc .........OK - Comparing phyf000.tile6.nc .........OK - Comparing phyf024.tile1.nc .........OK - Comparing phyf024.tile2.nc .........OK - Comparing phyf024.tile3.nc .........OK - Comparing phyf024.tile4.nc .........OK - Comparing phyf024.tile5.nc .........OK - Comparing phyf024.tile6.nc .........OK - Comparing dynf000.tile1.nc .........OK - Comparing dynf000.tile2.nc .........OK - Comparing dynf000.tile3.nc .........OK - Comparing dynf000.tile4.nc .........OK - Comparing dynf000.tile5.nc .........OK - Comparing dynf000.tile6.nc .........OK - Comparing dynf024.tile1.nc .........OK - Comparing dynf024.tile2.nc .........OK - Comparing dynf024.tile3.nc .........OK - Comparing dynf024.tile4.nc .........OK - Comparing dynf024.tile5.nc .........OK - Comparing dynf024.tile6.nc .........OK - Comparing RESTART/coupler.res .........OK - Comparing RESTART/fv_core.res.nc .........OK - Comparing RESTART/fv_core.res.tile1.nc .........OK - Comparing RESTART/fv_core.res.tile2.nc .........OK - Comparing RESTART/fv_core.res.tile3.nc .........OK - Comparing RESTART/fv_core.res.tile4.nc .........OK - Comparing RESTART/fv_core.res.tile5.nc .........OK - Comparing RESTART/fv_core.res.tile6.nc .........OK - Comparing RESTART/fv_srf_wnd.res.tile1.nc .........OK - Comparing RESTART/fv_srf_wnd.res.tile2.nc .........OK - Comparing RESTART/fv_srf_wnd.res.tile3.nc .........OK - Comparing RESTART/fv_srf_wnd.res.tile4.nc .........OK - Comparing RESTART/fv_srf_wnd.res.tile5.nc .........OK - Comparing RESTART/fv_srf_wnd.res.tile6.nc .........OK - Comparing RESTART/fv_tracer.res.tile1.nc .........OK - Comparing RESTART/fv_tracer.res.tile2.nc .........OK - Comparing RESTART/fv_tracer.res.tile3.nc .........OK - Comparing RESTART/fv_tracer.res.tile4.nc .........OK - Comparing RESTART/fv_tracer.res.tile5.nc .........OK - Comparing RESTART/fv_tracer.res.tile6.nc .........OK - Comparing RESTART/phy_data.tile1.nc .........OK - Comparing RESTART/phy_data.tile2.nc .........OK - Comparing RESTART/phy_data.tile3.nc .........OK - Comparing RESTART/phy_data.tile4.nc .........OK - Comparing RESTART/phy_data.tile5.nc .........OK - Comparing RESTART/phy_data.tile6.nc .........OK - Comparing RESTART/sfc_data.tile1.nc .........OK - Comparing RESTART/sfc_data.tile2.nc .........OK - Comparing RESTART/sfc_data.tile3.nc .........OK - Comparing RESTART/sfc_data.tile4.nc .........OK - Comparing RESTART/sfc_data.tile5.nc .........OK - Comparing RESTART/sfc_data.tile6.nc .........OK -Test 32bit fv3_ccpp_control PASS - - -baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200512/INTEL/fv3_control_debug -working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_UT/ut_72447/fv3_ccpp_control_debug_base -Checking test debug_base fv3_ccpp_control results .... - mkdir -p /scratch1/NCEPDEV/stmp4/Minsuk.Ji/FV3_UT/UNIT_TEST/fv3_control_debug/RESTART - Moving atmos_4xdaily.tile1.nc ..... Moving atmos_4xdaily.tile2.nc ..... Moving atmos_4xdaily.tile3.nc ..... Moving atmos_4xdaily.tile4.nc ..... Moving atmos_4xdaily.tile5.nc ..... Moving atmos_4xdaily.tile6.nc ..... Moving phyf000.tile1.nc ..... Moving phyf000.tile2.nc ..... Moving phyf000.tile3.nc ..... Moving phyf000.tile4.nc ..... Moving phyf000.tile5.nc ..... Moving phyf000.tile6.nc ..... Moving phyf024.tile1.nc ..... Moving phyf024.tile2.nc ..... Moving phyf024.tile3.nc ..... Moving phyf024.tile4.nc ..... Moving phyf024.tile5.nc ..... Moving phyf024.tile6.nc ..... Moving dynf000.tile1.nc ..... Moving dynf000.tile2.nc ..... Moving dynf000.tile3.nc ..... Moving dynf000.tile4.nc ..... Moving dynf000.tile5.nc ..... Moving dynf000.tile6.nc ..... Moving dynf024.tile1.nc ..... Moving dynf024.tile2.nc ..... Moving dynf024.tile3.nc ..... Moving dynf024.tile4.nc ..... Moving dynf024.tile5.nc ..... Moving dynf024.tile6.nc ..... Moving RESTART/coupler.res ..... Moving RESTART/fv_core.res.nc ..... Moving RESTART/fv_core.res.tile1.nc ..... Moving RESTART/fv_core.res.tile2.nc ..... Moving RESTART/fv_core.res.tile3.nc ..... Moving RESTART/fv_core.res.tile4.nc ..... Moving RESTART/fv_core.res.tile5.nc ..... Moving RESTART/fv_core.res.tile6.nc ..... Moving RESTART/fv_srf_wnd.res.tile1.nc ..... Moving RESTART/fv_srf_wnd.res.tile2.nc ..... Moving RESTART/fv_srf_wnd.res.tile3.nc ..... Moving RESTART/fv_srf_wnd.res.tile4.nc ..... Moving RESTART/fv_srf_wnd.res.tile5.nc ..... Moving RESTART/fv_srf_wnd.res.tile6.nc ..... Moving RESTART/fv_tracer.res.tile1.nc ..... Moving RESTART/fv_tracer.res.tile2.nc ..... Moving RESTART/fv_tracer.res.tile3.nc ..... Moving RESTART/fv_tracer.res.tile4.nc ..... Moving RESTART/fv_tracer.res.tile5.nc ..... Moving RESTART/fv_tracer.res.tile6.nc ..... Moving RESTART/phy_data.tile1.nc ..... Moving RESTART/phy_data.tile2.nc ..... Moving RESTART/phy_data.tile3.nc ..... Moving RESTART/phy_data.tile4.nc ..... Moving RESTART/phy_data.tile5.nc ..... Moving RESTART/phy_data.tile6.nc ..... Moving RESTART/sfc_data.tile1.nc ..... Moving RESTART/sfc_data.tile2.nc ..... Moving RESTART/sfc_data.tile3.nc ..... Moving RESTART/sfc_data.tile4.nc ..... Moving RESTART/sfc_data.tile5.nc ..... Moving RESTART/sfc_data.tile6.nc .....Test debug_base fv3_ccpp_control PASS - - -baseline dir = /scratch1/NCEPDEV/stmp4/Minsuk.Ji/FV3_UT/UNIT_TEST/fv3_control_debug -working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_UT/ut_72447/fv3_ccpp_control_debug -Checking test debug fv3_ccpp_control results .... - Comparing atmos_4xdaily.tile1.nc .........OK - Comparing atmos_4xdaily.tile2.nc .........OK - Comparing atmos_4xdaily.tile3.nc .........OK - Comparing atmos_4xdaily.tile4.nc .........OK - Comparing atmos_4xdaily.tile5.nc .........OK - Comparing atmos_4xdaily.tile6.nc .........OK - Comparing phyf000.tile1.nc .........OK - Comparing phyf000.tile2.nc .........OK - Comparing phyf000.tile3.nc .........OK - Comparing phyf000.tile4.nc .........OK - Comparing phyf000.tile5.nc .........OK - Comparing phyf000.tile6.nc .........OK - Comparing phyf024.tile1.nc .........OK - Comparing phyf024.tile2.nc .........OK - Comparing phyf024.tile3.nc .........OK - Comparing phyf024.tile4.nc .........OK - Comparing phyf024.tile5.nc .........OK - Comparing phyf024.tile6.nc .........OK - Comparing dynf000.tile1.nc .........OK - Comparing dynf000.tile2.nc .........OK - Comparing dynf000.tile3.nc .........OK - Comparing dynf000.tile4.nc .........OK - Comparing dynf000.tile5.nc .........OK - Comparing dynf000.tile6.nc .........OK - Comparing dynf024.tile1.nc .........OK - Comparing dynf024.tile2.nc .........OK - Comparing dynf024.tile3.nc .........OK - Comparing dynf024.tile4.nc .........OK - Comparing dynf024.tile5.nc .........OK - Comparing dynf024.tile6.nc .........OK - Comparing RESTART/coupler.res .........OK - Comparing RESTART/fv_core.res.nc .........OK - Comparing RESTART/fv_core.res.tile1.nc .........OK - Comparing RESTART/fv_core.res.tile2.nc .........OK - Comparing RESTART/fv_core.res.tile3.nc .........OK - Comparing RESTART/fv_core.res.tile4.nc .........OK - Comparing RESTART/fv_core.res.tile5.nc .........OK - Comparing RESTART/fv_core.res.tile6.nc .........OK - Comparing RESTART/fv_srf_wnd.res.tile1.nc .........OK - Comparing RESTART/fv_srf_wnd.res.tile2.nc .........OK - Comparing RESTART/fv_srf_wnd.res.tile3.nc .........OK - Comparing RESTART/fv_srf_wnd.res.tile4.nc .........OK - Comparing RESTART/fv_srf_wnd.res.tile5.nc .........OK - Comparing RESTART/fv_srf_wnd.res.tile6.nc .........OK - Comparing RESTART/fv_tracer.res.tile1.nc .........OK - Comparing RESTART/fv_tracer.res.tile2.nc .........OK - Comparing RESTART/fv_tracer.res.tile3.nc .........OK - Comparing RESTART/fv_tracer.res.tile4.nc .........OK - Comparing RESTART/fv_tracer.res.tile5.nc .........OK - Comparing RESTART/fv_tracer.res.tile6.nc .........OK - Comparing RESTART/phy_data.tile1.nc .........OK - Comparing RESTART/phy_data.tile2.nc .........OK - Comparing RESTART/phy_data.tile3.nc .........OK - Comparing RESTART/phy_data.tile4.nc .........OK - Comparing RESTART/phy_data.tile5.nc .........OK - Comparing RESTART/phy_data.tile6.nc .........OK - Comparing RESTART/sfc_data.tile1.nc .........OK - Comparing RESTART/sfc_data.tile2.nc .........OK - Comparing RESTART/sfc_data.tile3.nc .........OK - Comparing RESTART/sfc_data.tile4.nc .........OK - Comparing RESTART/sfc_data.tile5.nc .........OK - Comparing RESTART/sfc_data.tile6.nc .........OK -Test debug fv3_ccpp_control PASS +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_control_dbg_base +working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_UT/ut_269251/fv3_ccpp_control_dbg_base +Checking test dbg_base fv3_ccpp_control results .... +Moving baseline dbg_base fv3_ccpp_control files .... + mkdir -p /scratch1/NCEPDEV/stmp4/Minsuk.Ji/FV3_UT/UNIT_TEST/fv3_control_dbg_base/RESTART + Moving atmos_4xdaily.tile1.nc .........OK + Moving atmos_4xdaily.tile2.nc .........OK + Moving atmos_4xdaily.tile3.nc .........OK + Moving atmos_4xdaily.tile4.nc .........OK + Moving atmos_4xdaily.tile5.nc .........OK + Moving atmos_4xdaily.tile6.nc .........OK + Moving phyf000.tile1.nc .........OK + Moving phyf000.tile2.nc .........OK + Moving phyf000.tile3.nc .........OK + Moving phyf000.tile4.nc .........OK + Moving phyf000.tile5.nc .........OK + Moving phyf000.tile6.nc .........OK + Moving phyf024.tile1.nc .........OK + Moving phyf024.tile2.nc .........OK + Moving phyf024.tile3.nc .........OK + Moving phyf024.tile4.nc .........OK + Moving phyf024.tile5.nc .........OK + Moving phyf024.tile6.nc .........OK + Moving dynf000.tile1.nc .........OK + Moving dynf000.tile2.nc .........OK + Moving dynf000.tile3.nc .........OK + Moving dynf000.tile4.nc .........OK + Moving dynf000.tile5.nc .........OK + Moving dynf000.tile6.nc .........OK + Moving dynf024.tile1.nc .........OK + Moving dynf024.tile2.nc .........OK + Moving dynf024.tile3.nc .........OK + Moving dynf024.tile4.nc .........OK + Moving dynf024.tile5.nc .........OK + Moving dynf024.tile6.nc .........OK + Moving RESTART/coupler.res .........OK + Moving RESTART/fv_core.res.nc .........OK + Moving RESTART/fv_core.res.tile1.nc .........OK + Moving RESTART/fv_core.res.tile2.nc .........OK + Moving RESTART/fv_core.res.tile3.nc .........OK + Moving RESTART/fv_core.res.tile4.nc .........OK + Moving RESTART/fv_core.res.tile5.nc .........OK + Moving RESTART/fv_core.res.tile6.nc .........OK + Moving RESTART/fv_srf_wnd.res.tile1.nc .........OK + Moving RESTART/fv_srf_wnd.res.tile2.nc .........OK + Moving RESTART/fv_srf_wnd.res.tile3.nc .........OK + Moving RESTART/fv_srf_wnd.res.tile4.nc .........OK + Moving RESTART/fv_srf_wnd.res.tile5.nc .........OK + Moving RESTART/fv_srf_wnd.res.tile6.nc .........OK + Moving RESTART/fv_tracer.res.tile1.nc .........OK + Moving RESTART/fv_tracer.res.tile2.nc .........OK + Moving RESTART/fv_tracer.res.tile3.nc .........OK + Moving RESTART/fv_tracer.res.tile4.nc .........OK + Moving RESTART/fv_tracer.res.tile5.nc .........OK + Moving RESTART/fv_tracer.res.tile6.nc .........OK + Moving RESTART/phy_data.tile1.nc .........OK + Moving RESTART/phy_data.tile2.nc .........OK + Moving RESTART/phy_data.tile3.nc .........OK + Moving RESTART/phy_data.tile4.nc .........OK + Moving RESTART/phy_data.tile5.nc .........OK + Moving RESTART/phy_data.tile6.nc .........OK + Moving RESTART/sfc_data.tile1.nc .........OK + Moving RESTART/sfc_data.tile2.nc .........OK + Moving RESTART/sfc_data.tile3.nc .........OK + Moving RESTART/sfc_data.tile4.nc .........OK + Moving RESTART/sfc_data.tile5.nc .........OK + Moving RESTART/sfc_data.tile6.nc .........OK +Test dbg_base fv3_ccpp_control PASS -baseline dir = /scratch1/NCEPDEV/stmp4/Minsuk.Ji/FV3_UT/UNIT_TEST/fv3_control_std -working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_UT/ut_72447/fv3_ccpp_control_decomp -Checking test decomp fv3_ccpp_control results .... +baseline dir = /scratch1/NCEPDEV/stmp4/Minsuk.Ji/FV3_UT/UNIT_TEST/fv3_control_std_base +working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_UT/ut_269251/fv3_ccpp_control_dcp +Checking test dcp fv3_ccpp_control results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK Comparing atmos_4xdaily.tile3.nc .........OK @@ -214,11 +204,11 @@ Checking test decomp fv3_ccpp_control results .... Comparing RESTART/sfc_data.tile4.nc .........OK Comparing RESTART/sfc_data.tile5.nc .........OK Comparing RESTART/sfc_data.tile6.nc .........OK -Test decomp fv3_ccpp_control PASS +Test dcp fv3_ccpp_control PASS -baseline dir = /scratch1/NCEPDEV/stmp4/Minsuk.Ji/FV3_UT/UNIT_TEST/fv3_control_std -working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_UT/ut_72447/fv3_ccpp_control_mpi +baseline dir = /scratch1/NCEPDEV/stmp4/Minsuk.Ji/FV3_UT/UNIT_TEST/fv3_control_std_base +working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_UT/ut_269251/fv3_ccpp_control_mpi Checking test mpi fv3_ccpp_control results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -285,134 +275,129 @@ Checking test mpi fv3_ccpp_control results .... Test mpi fv3_ccpp_control PASS -baseline dir = /scratch1/NCEPDEV/stmp4/Minsuk.Ji/FV3_UT/UNIT_TEST/fv3_control_std -working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_UT/ut_72447/fv3_ccpp_control_restart -Checking test restart fv3_ccpp_control results .... - Comparing phyf024.tile1.nc .........OK - Comparing phyf024.tile2.nc .........OK - Comparing phyf024.tile3.nc .........OK - Comparing phyf024.tile4.nc .........OK - Comparing phyf024.tile5.nc .........OK - Comparing phyf024.tile6.nc .........OK - Comparing dynf024.tile1.nc .........OK - Comparing dynf024.tile2.nc .........OK - Comparing dynf024.tile3.nc .........OK - Comparing dynf024.tile4.nc .........OK - Comparing dynf024.tile5.nc .........OK - Comparing dynf024.tile6.nc .........OK +baseline dir = /scratch1/NCEPDEV/stmp4/Minsuk.Ji/FV3_UT/UNIT_TEST/fv3_control_std_base +working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_UT/ut_269251/fv3_ccpp_control_rst +Checking test rst fv3_ccpp_control results .... + Comparing phyf024.tile1.nc ............NOT OK + Comparing phyf024.tile2.nc ............NOT OK + Comparing phyf024.tile3.nc ............NOT OK + Comparing phyf024.tile4.nc ............NOT OK + Comparing phyf024.tile5.nc ............NOT OK + Comparing phyf024.tile6.nc ............NOT OK + Comparing dynf024.tile1.nc ............NOT OK + Comparing dynf024.tile2.nc ............NOT OK + Comparing dynf024.tile3.nc ............NOT OK + Comparing dynf024.tile4.nc ............NOT OK + Comparing dynf024.tile5.nc ............NOT OK + Comparing dynf024.tile6.nc ............NOT OK Comparing RESTART/coupler.res .........OK Comparing RESTART/fv_core.res.nc .........OK - Comparing RESTART/fv_core.res.tile1.nc .........OK - Comparing RESTART/fv_core.res.tile2.nc .........OK - Comparing RESTART/fv_core.res.tile3.nc .........OK - Comparing RESTART/fv_core.res.tile4.nc .........OK - Comparing RESTART/fv_core.res.tile5.nc .........OK - Comparing RESTART/fv_core.res.tile6.nc .........OK - Comparing RESTART/fv_srf_wnd.res.tile1.nc .........OK - Comparing RESTART/fv_srf_wnd.res.tile2.nc .........OK - Comparing RESTART/fv_srf_wnd.res.tile3.nc .........OK - Comparing RESTART/fv_srf_wnd.res.tile4.nc .........OK - Comparing RESTART/fv_srf_wnd.res.tile5.nc .........OK - Comparing RESTART/fv_srf_wnd.res.tile6.nc .........OK - Comparing RESTART/fv_tracer.res.tile1.nc .........OK - Comparing RESTART/fv_tracer.res.tile2.nc .........OK - Comparing RESTART/fv_tracer.res.tile3.nc .........OK - Comparing RESTART/fv_tracer.res.tile4.nc .........OK - Comparing RESTART/fv_tracer.res.tile5.nc .........OK - Comparing RESTART/fv_tracer.res.tile6.nc .........OK - Comparing RESTART/phy_data.tile1.nc .........OK - Comparing RESTART/phy_data.tile2.nc .........OK - Comparing RESTART/phy_data.tile3.nc .........OK - Comparing RESTART/phy_data.tile4.nc .........OK - Comparing RESTART/phy_data.tile5.nc .........OK - Comparing RESTART/phy_data.tile6.nc .........OK - Comparing RESTART/sfc_data.tile1.nc .........OK - Comparing RESTART/sfc_data.tile2.nc .........OK - Comparing RESTART/sfc_data.tile3.nc .........OK - Comparing RESTART/sfc_data.tile4.nc .........OK - Comparing RESTART/sfc_data.tile5.nc .........OK - Comparing RESTART/sfc_data.tile6.nc .........OK -Test restart fv3_ccpp_control PASS + Comparing RESTART/fv_core.res.tile1.nc ............NOT OK + Comparing RESTART/fv_core.res.tile2.nc ............NOT OK + Comparing RESTART/fv_core.res.tile3.nc ............NOT OK + Comparing RESTART/fv_core.res.tile4.nc ............NOT OK + Comparing RESTART/fv_core.res.tile5.nc ............NOT OK + Comparing RESTART/fv_core.res.tile6.nc ............NOT OK + Comparing RESTART/fv_srf_wnd.res.tile1.nc ............NOT OK + Comparing RESTART/fv_srf_wnd.res.tile2.nc ............NOT OK + Comparing RESTART/fv_srf_wnd.res.tile3.nc ............NOT OK + Comparing RESTART/fv_srf_wnd.res.tile4.nc ............NOT OK + Comparing RESTART/fv_srf_wnd.res.tile5.nc ............NOT OK + Comparing RESTART/fv_srf_wnd.res.tile6.nc ............NOT OK + Comparing RESTART/fv_tracer.res.tile1.nc ............NOT OK + Comparing RESTART/fv_tracer.res.tile2.nc ............NOT OK + Comparing RESTART/fv_tracer.res.tile3.nc ............NOT OK + Comparing RESTART/fv_tracer.res.tile4.nc ............NOT OK + Comparing RESTART/fv_tracer.res.tile5.nc ............NOT OK + Comparing RESTART/fv_tracer.res.tile6.nc ............NOT OK + Comparing RESTART/phy_data.tile1.nc ............NOT OK + Comparing RESTART/phy_data.tile2.nc ............NOT OK + Comparing RESTART/phy_data.tile3.nc ............NOT OK + Comparing RESTART/phy_data.tile4.nc ............NOT OK + Comparing RESTART/phy_data.tile5.nc ............NOT OK + Comparing RESTART/phy_data.tile6.nc ............NOT OK + Comparing RESTART/sfc_data.tile1.nc ............NOT OK + Comparing RESTART/sfc_data.tile2.nc ............NOT OK + Comparing RESTART/sfc_data.tile3.nc ............NOT OK + Comparing RESTART/sfc_data.tile4.nc ............NOT OK + Comparing RESTART/sfc_data.tile5.nc ............NOT OK + Comparing RESTART/sfc_data.tile6.nc ............NOT OK +Test rst fv3_ccpp_control FAIL -baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200512/INTEL/fv3_control_std -working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_UT/ut_72447/fv3_ccpp_control_std_base +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_control_std_base +working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_UT/ut_269251/fv3_ccpp_control_std_base Checking test std_base fv3_ccpp_control results .... - mkdir -p /scratch1/NCEPDEV/stmp4/Minsuk.Ji/FV3_UT/UNIT_TEST/fv3_control_std/RESTART - Moving atmos_4xdaily.tile1.nc ..... Moving atmos_4xdaily.tile2.nc ..... Moving atmos_4xdaily.tile3.nc ..... Moving atmos_4xdaily.tile4.nc ..... Moving atmos_4xdaily.tile5.nc ..... Moving atmos_4xdaily.tile6.nc ..... Moving phyf000.tile1.nc ..... Moving phyf000.tile2.nc ..... Moving phyf000.tile3.nc ..... Moving phyf000.tile4.nc ..... Moving phyf000.tile5.nc ..... Moving phyf000.tile6.nc ..... Moving phyf024.tile1.nc ..... Moving phyf024.tile2.nc ..... Moving phyf024.tile3.nc ..... Moving phyf024.tile4.nc ..... Moving phyf024.tile5.nc ..... Moving phyf024.tile6.nc ..... Moving dynf000.tile1.nc ..... Moving dynf000.tile2.nc ..... Moving dynf000.tile3.nc ..... Moving dynf000.tile4.nc ..... Moving dynf000.tile5.nc ..... Moving dynf000.tile6.nc ..... Moving dynf024.tile1.nc ..... Moving dynf024.tile2.nc ..... Moving dynf024.tile3.nc ..... Moving dynf024.tile4.nc ..... Moving dynf024.tile5.nc ..... Moving dynf024.tile6.nc ..... Moving RESTART/coupler.res ..... Moving RESTART/fv_core.res.nc ..... Moving RESTART/fv_core.res.tile1.nc ..... Moving RESTART/fv_core.res.tile2.nc ..... Moving RESTART/fv_core.res.tile3.nc ..... Moving RESTART/fv_core.res.tile4.nc ..... Moving RESTART/fv_core.res.tile5.nc ..... Moving RESTART/fv_core.res.tile6.nc ..... Moving RESTART/fv_srf_wnd.res.tile1.nc ..... Moving RESTART/fv_srf_wnd.res.tile2.nc ..... Moving RESTART/fv_srf_wnd.res.tile3.nc ..... Moving RESTART/fv_srf_wnd.res.tile4.nc ..... Moving RESTART/fv_srf_wnd.res.tile5.nc ..... Moving RESTART/fv_srf_wnd.res.tile6.nc ..... Moving RESTART/fv_tracer.res.tile1.nc ..... Moving RESTART/fv_tracer.res.tile2.nc ..... Moving RESTART/fv_tracer.res.tile3.nc ..... Moving RESTART/fv_tracer.res.tile4.nc ..... Moving RESTART/fv_tracer.res.tile5.nc ..... Moving RESTART/fv_tracer.res.tile6.nc ..... Moving RESTART/phy_data.tile1.nc ..... Moving RESTART/phy_data.tile2.nc ..... Moving RESTART/phy_data.tile3.nc ..... Moving RESTART/phy_data.tile4.nc ..... Moving RESTART/phy_data.tile5.nc ..... Moving RESTART/phy_data.tile6.nc ..... Moving RESTART/sfc_data.tile1.nc ..... Moving RESTART/sfc_data.tile2.nc ..... Moving RESTART/sfc_data.tile3.nc ..... Moving RESTART/sfc_data.tile4.nc ..... Moving RESTART/sfc_data.tile5.nc ..... Moving RESTART/sfc_data.tile6.nc .....Test std_base fv3_ccpp_control PASS - - -baseline dir = /scratch1/NCEPDEV/stmp4/Minsuk.Ji/FV3_UT/UNIT_TEST/fv3_control_std -working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_UT/ut_72447/fv3_ccpp_control_std -Checking test std fv3_ccpp_control results .... - Comparing atmos_4xdaily.tile1.nc .........OK - Comparing atmos_4xdaily.tile2.nc .........OK - Comparing atmos_4xdaily.tile3.nc .........OK - Comparing atmos_4xdaily.tile4.nc .........OK - Comparing atmos_4xdaily.tile5.nc .........OK - Comparing atmos_4xdaily.tile6.nc .........OK - Comparing phyf000.tile1.nc .........OK - Comparing phyf000.tile2.nc .........OK - Comparing phyf000.tile3.nc .........OK - Comparing phyf000.tile4.nc .........OK - Comparing phyf000.tile5.nc .........OK - Comparing phyf000.tile6.nc .........OK - Comparing phyf024.tile1.nc .........OK - Comparing phyf024.tile2.nc .........OK - Comparing phyf024.tile3.nc .........OK - Comparing phyf024.tile4.nc .........OK - Comparing phyf024.tile5.nc .........OK - Comparing phyf024.tile6.nc .........OK - Comparing dynf000.tile1.nc .........OK - Comparing dynf000.tile2.nc .........OK - Comparing dynf000.tile3.nc .........OK - Comparing dynf000.tile4.nc .........OK - Comparing dynf000.tile5.nc .........OK - Comparing dynf000.tile6.nc .........OK - Comparing dynf024.tile1.nc .........OK - Comparing dynf024.tile2.nc .........OK - Comparing dynf024.tile3.nc .........OK - Comparing dynf024.tile4.nc .........OK - Comparing dynf024.tile5.nc .........OK - Comparing dynf024.tile6.nc .........OK - Comparing RESTART/coupler.res .........OK - Comparing RESTART/fv_core.res.nc .........OK - Comparing RESTART/fv_core.res.tile1.nc .........OK - Comparing RESTART/fv_core.res.tile2.nc .........OK - Comparing RESTART/fv_core.res.tile3.nc .........OK - Comparing RESTART/fv_core.res.tile4.nc .........OK - Comparing RESTART/fv_core.res.tile5.nc .........OK - Comparing RESTART/fv_core.res.tile6.nc .........OK - Comparing RESTART/fv_srf_wnd.res.tile1.nc .........OK - Comparing RESTART/fv_srf_wnd.res.tile2.nc .........OK - Comparing RESTART/fv_srf_wnd.res.tile3.nc .........OK - Comparing RESTART/fv_srf_wnd.res.tile4.nc .........OK - Comparing RESTART/fv_srf_wnd.res.tile5.nc .........OK - Comparing RESTART/fv_srf_wnd.res.tile6.nc .........OK - Comparing RESTART/fv_tracer.res.tile1.nc .........OK - Comparing RESTART/fv_tracer.res.tile2.nc .........OK - Comparing RESTART/fv_tracer.res.tile3.nc .........OK - Comparing RESTART/fv_tracer.res.tile4.nc .........OK - Comparing RESTART/fv_tracer.res.tile5.nc .........OK - Comparing RESTART/fv_tracer.res.tile6.nc .........OK - Comparing RESTART/phy_data.tile1.nc .........OK - Comparing RESTART/phy_data.tile2.nc .........OK - Comparing RESTART/phy_data.tile3.nc .........OK - Comparing RESTART/phy_data.tile4.nc .........OK - Comparing RESTART/phy_data.tile5.nc .........OK - Comparing RESTART/phy_data.tile6.nc .........OK - Comparing RESTART/sfc_data.tile1.nc .........OK - Comparing RESTART/sfc_data.tile2.nc .........OK - Comparing RESTART/sfc_data.tile3.nc .........OK - Comparing RESTART/sfc_data.tile4.nc .........OK - Comparing RESTART/sfc_data.tile5.nc .........OK - Comparing RESTART/sfc_data.tile6.nc .........OK -Test std fv3_ccpp_control PASS +Moving baseline std_base fv3_ccpp_control files .... + mkdir -p /scratch1/NCEPDEV/stmp4/Minsuk.Ji/FV3_UT/UNIT_TEST/fv3_control_std_base/RESTART + Moving atmos_4xdaily.tile1.nc .........OK + Moving atmos_4xdaily.tile2.nc .........OK + Moving atmos_4xdaily.tile3.nc .........OK + Moving atmos_4xdaily.tile4.nc .........OK + Moving atmos_4xdaily.tile5.nc .........OK + Moving atmos_4xdaily.tile6.nc .........OK + Moving phyf000.tile1.nc .........OK + Moving phyf000.tile2.nc .........OK + Moving phyf000.tile3.nc .........OK + Moving phyf000.tile4.nc .........OK + Moving phyf000.tile5.nc .........OK + Moving phyf000.tile6.nc .........OK + Moving phyf024.tile1.nc .........OK + Moving phyf024.tile2.nc .........OK + Moving phyf024.tile3.nc .........OK + Moving phyf024.tile4.nc .........OK + Moving phyf024.tile5.nc .........OK + Moving phyf024.tile6.nc .........OK + Moving dynf000.tile1.nc .........OK + Moving dynf000.tile2.nc .........OK + Moving dynf000.tile3.nc .........OK + Moving dynf000.tile4.nc .........OK + Moving dynf000.tile5.nc .........OK + Moving dynf000.tile6.nc .........OK + Moving dynf024.tile1.nc .........OK + Moving dynf024.tile2.nc .........OK + Moving dynf024.tile3.nc .........OK + Moving dynf024.tile4.nc .........OK + Moving dynf024.tile5.nc .........OK + Moving dynf024.tile6.nc .........OK + Moving RESTART/coupler.res .........OK + Moving RESTART/fv_core.res.nc .........OK + Moving RESTART/fv_core.res.tile1.nc .........OK + Moving RESTART/fv_core.res.tile2.nc .........OK + Moving RESTART/fv_core.res.tile3.nc .........OK + Moving RESTART/fv_core.res.tile4.nc .........OK + Moving RESTART/fv_core.res.tile5.nc .........OK + Moving RESTART/fv_core.res.tile6.nc .........OK + Moving RESTART/fv_srf_wnd.res.tile1.nc .........OK + Moving RESTART/fv_srf_wnd.res.tile2.nc .........OK + Moving RESTART/fv_srf_wnd.res.tile3.nc .........OK + Moving RESTART/fv_srf_wnd.res.tile4.nc .........OK + Moving RESTART/fv_srf_wnd.res.tile5.nc .........OK + Moving RESTART/fv_srf_wnd.res.tile6.nc .........OK + Moving RESTART/fv_tracer.res.tile1.nc .........OK + Moving RESTART/fv_tracer.res.tile2.nc .........OK + Moving RESTART/fv_tracer.res.tile3.nc .........OK + Moving RESTART/fv_tracer.res.tile4.nc .........OK + Moving RESTART/fv_tracer.res.tile5.nc .........OK + Moving RESTART/fv_tracer.res.tile6.nc .........OK + Moving RESTART/phy_data.tile1.nc .........OK + Moving RESTART/phy_data.tile2.nc .........OK + Moving RESTART/phy_data.tile3.nc .........OK + Moving RESTART/phy_data.tile4.nc .........OK + Moving RESTART/phy_data.tile5.nc .........OK + Moving RESTART/phy_data.tile6.nc .........OK + Moving RESTART/sfc_data.tile1.nc .........OK + Moving RESTART/sfc_data.tile2.nc .........OK + Moving RESTART/sfc_data.tile3.nc .........OK + Moving RESTART/sfc_data.tile4.nc .........OK + Moving RESTART/sfc_data.tile5.nc .........OK + Moving RESTART/sfc_data.tile6.nc .........OK +Test std_base fv3_ccpp_control PASS -baseline dir = /scratch1/NCEPDEV/stmp4/Minsuk.Ji/FV3_UT/UNIT_TEST/fv3_control_std -working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_UT/ut_72447/fv3_ccpp_control_thread -Checking test thread fv3_ccpp_control results .... +baseline dir = /scratch1/NCEPDEV/stmp4/Minsuk.Ji/FV3_UT/UNIT_TEST/fv3_control_std_base +working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/FV3_UT/ut_269251/fv3_ccpp_control_thr +Checking test thr fv3_ccpp_control results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK Comparing atmos_4xdaily.tile3.nc .........OK @@ -475,8 +460,11 @@ Checking test thread fv3_ccpp_control results .... Comparing RESTART/sfc_data.tile4.nc .........OK Comparing RESTART/sfc_data.tile5.nc .........OK Comparing RESTART/sfc_data.tile6.nc .........OK -Test thread fv3_ccpp_control PASS +Test thr fv3_ccpp_control PASS -UNIT TEST WAS SUCCESSFUL -Wed Jun 3 17:49:52 UTC 2020 -Elapsed time: 01h:11m:03s. Have a nice day! +FAILED TESTS: +Test rst fv3_ccpp_control failed +Test rst fv3_ccpp_control failed +UNIT TEST FAILED +Tue Sep 8 20:50:57 UTC 2020 +Elapsed time: 00h:15m:11s. Have a nice day! diff --git a/tests/UnitTests_orion.intel.log b/tests/UnitTests_orion.intel.log new file mode 100644 index 0000000000..6a5a492851 --- /dev/null +++ b/tests/UnitTests_orion.intel.log @@ -0,0 +1,470 @@ + +baseline dir = /work/noaa/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_control_bit_base +working dir = /work/noaa/stmp/jminsuk/stmp/jminsuk/FV3_UT/ut_100015/fv3_ccpp_control_bit_base +Checking test bit_base fv3_ccpp_control results .... +Moving baseline bit_base fv3_ccpp_control files .... + mkdir -p /work/noaa/stmp/jminsuk/stmp/jminsuk/FV3_UT/UNIT_TEST/fv3_control_bit_base/RESTART + Moving atmos_4xdaily.tile1.nc .........OK + Moving atmos_4xdaily.tile2.nc .........OK + Moving atmos_4xdaily.tile3.nc .........OK + Moving atmos_4xdaily.tile4.nc .........OK + Moving atmos_4xdaily.tile5.nc .........OK + Moving atmos_4xdaily.tile6.nc .........OK + Moving phyf000.tile1.nc .........OK + Moving phyf000.tile2.nc .........OK + Moving phyf000.tile3.nc .........OK + Moving phyf000.tile4.nc .........OK + Moving phyf000.tile5.nc .........OK + Moving phyf000.tile6.nc .........OK + Moving phyf024.tile1.nc .........OK + Moving phyf024.tile2.nc .........OK + Moving phyf024.tile3.nc .........OK + Moving phyf024.tile4.nc .........OK + Moving phyf024.tile5.nc .........OK + Moving phyf024.tile6.nc .........OK + Moving dynf000.tile1.nc .........OK + Moving dynf000.tile2.nc .........OK + Moving dynf000.tile3.nc .........OK + Moving dynf000.tile4.nc .........OK + Moving dynf000.tile5.nc .........OK + Moving dynf000.tile6.nc .........OK + Moving dynf024.tile1.nc .........OK + Moving dynf024.tile2.nc .........OK + Moving dynf024.tile3.nc .........OK + Moving dynf024.tile4.nc .........OK + Moving dynf024.tile5.nc .........OK + Moving dynf024.tile6.nc .........OK + Moving RESTART/coupler.res .........OK + Moving RESTART/fv_core.res.nc .........OK + Moving RESTART/fv_core.res.tile1.nc .........OK + Moving RESTART/fv_core.res.tile2.nc .........OK + Moving RESTART/fv_core.res.tile3.nc .........OK + Moving RESTART/fv_core.res.tile4.nc .........OK + Moving RESTART/fv_core.res.tile5.nc .........OK + Moving RESTART/fv_core.res.tile6.nc .........OK + Moving RESTART/fv_srf_wnd.res.tile1.nc .........OK + Moving RESTART/fv_srf_wnd.res.tile2.nc .........OK + Moving RESTART/fv_srf_wnd.res.tile3.nc .........OK + Moving RESTART/fv_srf_wnd.res.tile4.nc .........OK + Moving RESTART/fv_srf_wnd.res.tile5.nc .........OK + Moving RESTART/fv_srf_wnd.res.tile6.nc .........OK + Moving RESTART/fv_tracer.res.tile1.nc .........OK + Moving RESTART/fv_tracer.res.tile2.nc .........OK + Moving RESTART/fv_tracer.res.tile3.nc .........OK + Moving RESTART/fv_tracer.res.tile4.nc .........OK + Moving RESTART/fv_tracer.res.tile5.nc .........OK + Moving RESTART/fv_tracer.res.tile6.nc .........OK + Moving RESTART/phy_data.tile1.nc .........OK + Moving RESTART/phy_data.tile2.nc .........OK + Moving RESTART/phy_data.tile3.nc .........OK + Moving RESTART/phy_data.tile4.nc .........OK + Moving RESTART/phy_data.tile5.nc .........OK + Moving RESTART/phy_data.tile6.nc .........OK + Moving RESTART/sfc_data.tile1.nc .........OK + Moving RESTART/sfc_data.tile2.nc .........OK + Moving RESTART/sfc_data.tile3.nc .........OK + Moving RESTART/sfc_data.tile4.nc .........OK + Moving RESTART/sfc_data.tile5.nc .........OK + Moving RESTART/sfc_data.tile6.nc .........OK +Test bit_base fv3_ccpp_control PASS + + +baseline dir = /work/noaa/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_control_dbg_base +working dir = /work/noaa/stmp/jminsuk/stmp/jminsuk/FV3_UT/ut_100015/fv3_ccpp_control_dbg_base +Checking test dbg_base fv3_ccpp_control results .... +Moving baseline dbg_base fv3_ccpp_control files .... + mkdir -p /work/noaa/stmp/jminsuk/stmp/jminsuk/FV3_UT/UNIT_TEST/fv3_control_dbg_base/RESTART + Moving atmos_4xdaily.tile1.nc .........OK + Moving atmos_4xdaily.tile2.nc .........OK + Moving atmos_4xdaily.tile3.nc .........OK + Moving atmos_4xdaily.tile4.nc .........OK + Moving atmos_4xdaily.tile5.nc .........OK + Moving atmos_4xdaily.tile6.nc .........OK + Moving phyf000.tile1.nc .........OK + Moving phyf000.tile2.nc .........OK + Moving phyf000.tile3.nc .........OK + Moving phyf000.tile4.nc .........OK + Moving phyf000.tile5.nc .........OK + Moving phyf000.tile6.nc .........OK + Moving phyf024.tile1.nc .........OK + Moving phyf024.tile2.nc .........OK + Moving phyf024.tile3.nc .........OK + Moving phyf024.tile4.nc .........OK + Moving phyf024.tile5.nc .........OK + Moving phyf024.tile6.nc .........OK + Moving dynf000.tile1.nc .........OK + Moving dynf000.tile2.nc .........OK + Moving dynf000.tile3.nc .........OK + Moving dynf000.tile4.nc .........OK + Moving dynf000.tile5.nc .........OK + Moving dynf000.tile6.nc .........OK + Moving dynf024.tile1.nc .........OK + Moving dynf024.tile2.nc .........OK + Moving dynf024.tile3.nc .........OK + Moving dynf024.tile4.nc .........OK + Moving dynf024.tile5.nc .........OK + Moving dynf024.tile6.nc .........OK + Moving RESTART/coupler.res .........OK + Moving RESTART/fv_core.res.nc .........OK + Moving RESTART/fv_core.res.tile1.nc .........OK + Moving RESTART/fv_core.res.tile2.nc .........OK + Moving RESTART/fv_core.res.tile3.nc .........OK + Moving RESTART/fv_core.res.tile4.nc .........OK + Moving RESTART/fv_core.res.tile5.nc .........OK + Moving RESTART/fv_core.res.tile6.nc .........OK + Moving RESTART/fv_srf_wnd.res.tile1.nc .........OK + Moving RESTART/fv_srf_wnd.res.tile2.nc .........OK + Moving RESTART/fv_srf_wnd.res.tile3.nc .........OK + Moving RESTART/fv_srf_wnd.res.tile4.nc .........OK + Moving RESTART/fv_srf_wnd.res.tile5.nc .........OK + Moving RESTART/fv_srf_wnd.res.tile6.nc .........OK + Moving RESTART/fv_tracer.res.tile1.nc .........OK + Moving RESTART/fv_tracer.res.tile2.nc .........OK + Moving RESTART/fv_tracer.res.tile3.nc .........OK + Moving RESTART/fv_tracer.res.tile4.nc .........OK + Moving RESTART/fv_tracer.res.tile5.nc .........OK + Moving RESTART/fv_tracer.res.tile6.nc .........OK + Moving RESTART/phy_data.tile1.nc .........OK + Moving RESTART/phy_data.tile2.nc .........OK + Moving RESTART/phy_data.tile3.nc .........OK + Moving RESTART/phy_data.tile4.nc .........OK + Moving RESTART/phy_data.tile5.nc .........OK + Moving RESTART/phy_data.tile6.nc .........OK + Moving RESTART/sfc_data.tile1.nc .........OK + Moving RESTART/sfc_data.tile2.nc .........OK + Moving RESTART/sfc_data.tile3.nc .........OK + Moving RESTART/sfc_data.tile4.nc .........OK + Moving RESTART/sfc_data.tile5.nc .........OK + Moving RESTART/sfc_data.tile6.nc .........OK +Test dbg_base fv3_ccpp_control PASS + + +baseline dir = /work/noaa/stmp/jminsuk/stmp/jminsuk/FV3_UT/UNIT_TEST/fv3_control_std_base +working dir = /work/noaa/stmp/jminsuk/stmp/jminsuk/FV3_UT/ut_100015/fv3_ccpp_control_dcp +Checking test dcp fv3_ccpp_control results .... + Comparing atmos_4xdaily.tile1.nc .........OK + Comparing atmos_4xdaily.tile2.nc .........OK + Comparing atmos_4xdaily.tile3.nc .........OK + Comparing atmos_4xdaily.tile4.nc .........OK + Comparing atmos_4xdaily.tile5.nc .........OK + Comparing atmos_4xdaily.tile6.nc .........OK + Comparing phyf000.tile1.nc .........OK + Comparing phyf000.tile2.nc .........OK + Comparing phyf000.tile3.nc .........OK + Comparing phyf000.tile4.nc .........OK + Comparing phyf000.tile5.nc .........OK + Comparing phyf000.tile6.nc .........OK + Comparing phyf024.tile1.nc .........OK + Comparing phyf024.tile2.nc .........OK + Comparing phyf024.tile3.nc .........OK + Comparing phyf024.tile4.nc .........OK + Comparing phyf024.tile5.nc .........OK + Comparing phyf024.tile6.nc .........OK + Comparing dynf000.tile1.nc .........OK + Comparing dynf000.tile2.nc .........OK + Comparing dynf000.tile3.nc .........OK + Comparing dynf000.tile4.nc .........OK + Comparing dynf000.tile5.nc .........OK + Comparing dynf000.tile6.nc .........OK + Comparing dynf024.tile1.nc .........OK + Comparing dynf024.tile2.nc .........OK + Comparing dynf024.tile3.nc .........OK + Comparing dynf024.tile4.nc .........OK + Comparing dynf024.tile5.nc .........OK + Comparing dynf024.tile6.nc .........OK + Comparing RESTART/coupler.res .........OK + Comparing RESTART/fv_core.res.nc .........OK + Comparing RESTART/fv_core.res.tile1.nc .........OK + Comparing RESTART/fv_core.res.tile2.nc .........OK + Comparing RESTART/fv_core.res.tile3.nc .........OK + Comparing RESTART/fv_core.res.tile4.nc .........OK + Comparing RESTART/fv_core.res.tile5.nc .........OK + Comparing RESTART/fv_core.res.tile6.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile1.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile2.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile3.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile4.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile5.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile6.nc .........OK + Comparing RESTART/fv_tracer.res.tile1.nc .........OK + Comparing RESTART/fv_tracer.res.tile2.nc .........OK + Comparing RESTART/fv_tracer.res.tile3.nc .........OK + Comparing RESTART/fv_tracer.res.tile4.nc .........OK + Comparing RESTART/fv_tracer.res.tile5.nc .........OK + Comparing RESTART/fv_tracer.res.tile6.nc .........OK + Comparing RESTART/phy_data.tile1.nc .........OK + Comparing RESTART/phy_data.tile2.nc .........OK + Comparing RESTART/phy_data.tile3.nc .........OK + Comparing RESTART/phy_data.tile4.nc .........OK + Comparing RESTART/phy_data.tile5.nc .........OK + Comparing RESTART/phy_data.tile6.nc .........OK + Comparing RESTART/sfc_data.tile1.nc .........OK + Comparing RESTART/sfc_data.tile2.nc .........OK + Comparing RESTART/sfc_data.tile3.nc .........OK + Comparing RESTART/sfc_data.tile4.nc .........OK + Comparing RESTART/sfc_data.tile5.nc .........OK + Comparing RESTART/sfc_data.tile6.nc .........OK +Test dcp fv3_ccpp_control PASS + + +baseline dir = /work/noaa/stmp/jminsuk/stmp/jminsuk/FV3_UT/UNIT_TEST/fv3_control_std_base +working dir = /work/noaa/stmp/jminsuk/stmp/jminsuk/FV3_UT/ut_100015/fv3_ccpp_control_mpi +Checking test mpi fv3_ccpp_control results .... + Comparing atmos_4xdaily.tile1.nc .........OK + Comparing atmos_4xdaily.tile2.nc .........OK + Comparing atmos_4xdaily.tile3.nc .........OK + Comparing atmos_4xdaily.tile4.nc .........OK + Comparing atmos_4xdaily.tile5.nc .........OK + Comparing atmos_4xdaily.tile6.nc .........OK + Comparing phyf000.tile1.nc .........OK + Comparing phyf000.tile2.nc .........OK + Comparing phyf000.tile3.nc .........OK + Comparing phyf000.tile4.nc .........OK + Comparing phyf000.tile5.nc .........OK + Comparing phyf000.tile6.nc .........OK + Comparing phyf024.tile1.nc .........OK + Comparing phyf024.tile2.nc .........OK + Comparing phyf024.tile3.nc .........OK + Comparing phyf024.tile4.nc .........OK + Comparing phyf024.tile5.nc .........OK + Comparing phyf024.tile6.nc .........OK + Comparing dynf000.tile1.nc .........OK + Comparing dynf000.tile2.nc .........OK + Comparing dynf000.tile3.nc .........OK + Comparing dynf000.tile4.nc .........OK + Comparing dynf000.tile5.nc .........OK + Comparing dynf000.tile6.nc .........OK + Comparing dynf024.tile1.nc .........OK + Comparing dynf024.tile2.nc .........OK + Comparing dynf024.tile3.nc .........OK + Comparing dynf024.tile4.nc .........OK + Comparing dynf024.tile5.nc .........OK + Comparing dynf024.tile6.nc .........OK + Comparing RESTART/coupler.res .........OK + Comparing RESTART/fv_core.res.nc .........OK + Comparing RESTART/fv_core.res.tile1.nc .........OK + Comparing RESTART/fv_core.res.tile2.nc .........OK + Comparing RESTART/fv_core.res.tile3.nc .........OK + Comparing RESTART/fv_core.res.tile4.nc .........OK + Comparing RESTART/fv_core.res.tile5.nc .........OK + Comparing RESTART/fv_core.res.tile6.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile1.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile2.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile3.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile4.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile5.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile6.nc .........OK + Comparing RESTART/fv_tracer.res.tile1.nc .........OK + Comparing RESTART/fv_tracer.res.tile2.nc .........OK + Comparing RESTART/fv_tracer.res.tile3.nc .........OK + Comparing RESTART/fv_tracer.res.tile4.nc .........OK + Comparing RESTART/fv_tracer.res.tile5.nc .........OK + Comparing RESTART/fv_tracer.res.tile6.nc .........OK + Comparing RESTART/phy_data.tile1.nc .........OK + Comparing RESTART/phy_data.tile2.nc .........OK + Comparing RESTART/phy_data.tile3.nc .........OK + Comparing RESTART/phy_data.tile4.nc .........OK + Comparing RESTART/phy_data.tile5.nc .........OK + Comparing RESTART/phy_data.tile6.nc .........OK + Comparing RESTART/sfc_data.tile1.nc .........OK + Comparing RESTART/sfc_data.tile2.nc .........OK + Comparing RESTART/sfc_data.tile3.nc .........OK + Comparing RESTART/sfc_data.tile4.nc .........OK + Comparing RESTART/sfc_data.tile5.nc .........OK + Comparing RESTART/sfc_data.tile6.nc .........OK +Test mpi fv3_ccpp_control PASS + + +baseline dir = /work/noaa/stmp/jminsuk/stmp/jminsuk/FV3_UT/UNIT_TEST/fv3_control_std_base +working dir = /work/noaa/stmp/jminsuk/stmp/jminsuk/FV3_UT/ut_100015/fv3_ccpp_control_rst +Checking test rst fv3_ccpp_control results .... + Comparing phyf024.tile1.nc ............NOT OK + Comparing phyf024.tile2.nc ............NOT OK + Comparing phyf024.tile3.nc ............NOT OK + Comparing phyf024.tile4.nc ............NOT OK + Comparing phyf024.tile5.nc ............NOT OK + Comparing phyf024.tile6.nc ............NOT OK + Comparing dynf024.tile1.nc ............NOT OK + Comparing dynf024.tile2.nc ............NOT OK + Comparing dynf024.tile3.nc ............NOT OK + Comparing dynf024.tile4.nc ............NOT OK + Comparing dynf024.tile5.nc ............NOT OK + Comparing dynf024.tile6.nc ............NOT OK + Comparing RESTART/coupler.res .........OK + Comparing RESTART/fv_core.res.nc .........OK + Comparing RESTART/fv_core.res.tile1.nc ............NOT OK + Comparing RESTART/fv_core.res.tile2.nc ............NOT OK + Comparing RESTART/fv_core.res.tile3.nc ............NOT OK + Comparing RESTART/fv_core.res.tile4.nc ............NOT OK + Comparing RESTART/fv_core.res.tile5.nc ............NOT OK + Comparing RESTART/fv_core.res.tile6.nc ............NOT OK + Comparing RESTART/fv_srf_wnd.res.tile1.nc ............NOT OK + Comparing RESTART/fv_srf_wnd.res.tile2.nc ............NOT OK + Comparing RESTART/fv_srf_wnd.res.tile3.nc ............NOT OK + Comparing RESTART/fv_srf_wnd.res.tile4.nc ............NOT OK + Comparing RESTART/fv_srf_wnd.res.tile5.nc ............NOT OK + Comparing RESTART/fv_srf_wnd.res.tile6.nc ............NOT OK + Comparing RESTART/fv_tracer.res.tile1.nc ............NOT OK + Comparing RESTART/fv_tracer.res.tile2.nc ............NOT OK + Comparing RESTART/fv_tracer.res.tile3.nc ............NOT OK + Comparing RESTART/fv_tracer.res.tile4.nc ............NOT OK + Comparing RESTART/fv_tracer.res.tile5.nc ............NOT OK + Comparing RESTART/fv_tracer.res.tile6.nc ............NOT OK + Comparing RESTART/phy_data.tile1.nc ............NOT OK + Comparing RESTART/phy_data.tile2.nc ............NOT OK + Comparing RESTART/phy_data.tile3.nc ............NOT OK + Comparing RESTART/phy_data.tile4.nc ............NOT OK + Comparing RESTART/phy_data.tile5.nc ............NOT OK + Comparing RESTART/phy_data.tile6.nc ............NOT OK + Comparing RESTART/sfc_data.tile1.nc ............NOT OK + Comparing RESTART/sfc_data.tile2.nc ............NOT OK + Comparing RESTART/sfc_data.tile3.nc ............NOT OK + Comparing RESTART/sfc_data.tile4.nc ............NOT OK + Comparing RESTART/sfc_data.tile5.nc ............NOT OK + Comparing RESTART/sfc_data.tile6.nc ............NOT OK +Test rst fv3_ccpp_control FAIL + + +baseline dir = /work/noaa/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_control_std_base +working dir = /work/noaa/stmp/jminsuk/stmp/jminsuk/FV3_UT/ut_100015/fv3_ccpp_control_std_base +Checking test std_base fv3_ccpp_control results .... +Moving baseline std_base fv3_ccpp_control files .... + mkdir -p /work/noaa/stmp/jminsuk/stmp/jminsuk/FV3_UT/UNIT_TEST/fv3_control_std_base/RESTART + Moving atmos_4xdaily.tile1.nc .........OK + Moving atmos_4xdaily.tile2.nc .........OK + Moving atmos_4xdaily.tile3.nc .........OK + Moving atmos_4xdaily.tile4.nc .........OK + Moving atmos_4xdaily.tile5.nc .........OK + Moving atmos_4xdaily.tile6.nc .........OK + Moving phyf000.tile1.nc .........OK + Moving phyf000.tile2.nc .........OK + Moving phyf000.tile3.nc .........OK + Moving phyf000.tile4.nc .........OK + Moving phyf000.tile5.nc .........OK + Moving phyf000.tile6.nc .........OK + Moving phyf024.tile1.nc .........OK + Moving phyf024.tile2.nc .........OK + Moving phyf024.tile3.nc .........OK + Moving phyf024.tile4.nc .........OK + Moving phyf024.tile5.nc .........OK + Moving phyf024.tile6.nc .........OK + Moving dynf000.tile1.nc .........OK + Moving dynf000.tile2.nc .........OK + Moving dynf000.tile3.nc .........OK + Moving dynf000.tile4.nc .........OK + Moving dynf000.tile5.nc .........OK + Moving dynf000.tile6.nc .........OK + Moving dynf024.tile1.nc .........OK + Moving dynf024.tile2.nc .........OK + Moving dynf024.tile3.nc .........OK + Moving dynf024.tile4.nc .........OK + Moving dynf024.tile5.nc .........OK + Moving dynf024.tile6.nc .........OK + Moving RESTART/coupler.res .........OK + Moving RESTART/fv_core.res.nc .........OK + Moving RESTART/fv_core.res.tile1.nc .........OK + Moving RESTART/fv_core.res.tile2.nc .........OK + Moving RESTART/fv_core.res.tile3.nc .........OK + Moving RESTART/fv_core.res.tile4.nc .........OK + Moving RESTART/fv_core.res.tile5.nc .........OK + Moving RESTART/fv_core.res.tile6.nc .........OK + Moving RESTART/fv_srf_wnd.res.tile1.nc .........OK + Moving RESTART/fv_srf_wnd.res.tile2.nc .........OK + Moving RESTART/fv_srf_wnd.res.tile3.nc .........OK + Moving RESTART/fv_srf_wnd.res.tile4.nc .........OK + Moving RESTART/fv_srf_wnd.res.tile5.nc .........OK + Moving RESTART/fv_srf_wnd.res.tile6.nc .........OK + Moving RESTART/fv_tracer.res.tile1.nc .........OK + Moving RESTART/fv_tracer.res.tile2.nc .........OK + Moving RESTART/fv_tracer.res.tile3.nc .........OK + Moving RESTART/fv_tracer.res.tile4.nc .........OK + Moving RESTART/fv_tracer.res.tile5.nc .........OK + Moving RESTART/fv_tracer.res.tile6.nc .........OK + Moving RESTART/phy_data.tile1.nc .........OK + Moving RESTART/phy_data.tile2.nc .........OK + Moving RESTART/phy_data.tile3.nc .........OK + Moving RESTART/phy_data.tile4.nc .........OK + Moving RESTART/phy_data.tile5.nc .........OK + Moving RESTART/phy_data.tile6.nc .........OK + Moving RESTART/sfc_data.tile1.nc .........OK + Moving RESTART/sfc_data.tile2.nc .........OK + Moving RESTART/sfc_data.tile3.nc .........OK + Moving RESTART/sfc_data.tile4.nc .........OK + Moving RESTART/sfc_data.tile5.nc .........OK + Moving RESTART/sfc_data.tile6.nc .........OK +Test std_base fv3_ccpp_control PASS + + +baseline dir = /work/noaa/stmp/jminsuk/stmp/jminsuk/FV3_UT/UNIT_TEST/fv3_control_std_base +working dir = /work/noaa/stmp/jminsuk/stmp/jminsuk/FV3_UT/ut_100015/fv3_ccpp_control_thr +Checking test thr fv3_ccpp_control results .... + Comparing atmos_4xdaily.tile1.nc .........OK + Comparing atmos_4xdaily.tile2.nc .........OK + Comparing atmos_4xdaily.tile3.nc .........OK + Comparing atmos_4xdaily.tile4.nc .........OK + Comparing atmos_4xdaily.tile5.nc .........OK + Comparing atmos_4xdaily.tile6.nc .........OK + Comparing phyf000.tile1.nc .........OK + Comparing phyf000.tile2.nc .........OK + Comparing phyf000.tile3.nc .........OK + Comparing phyf000.tile4.nc .........OK + Comparing phyf000.tile5.nc .........OK + Comparing phyf000.tile6.nc .........OK + Comparing phyf024.tile1.nc .........OK + Comparing phyf024.tile2.nc .........OK + Comparing phyf024.tile3.nc .........OK + Comparing phyf024.tile4.nc .........OK + Comparing phyf024.tile5.nc .........OK + Comparing phyf024.tile6.nc .........OK + Comparing dynf000.tile1.nc .........OK + Comparing dynf000.tile2.nc .........OK + Comparing dynf000.tile3.nc .........OK + Comparing dynf000.tile4.nc .........OK + Comparing dynf000.tile5.nc .........OK + Comparing dynf000.tile6.nc .........OK + Comparing dynf024.tile1.nc .........OK + Comparing dynf024.tile2.nc .........OK + Comparing dynf024.tile3.nc .........OK + Comparing dynf024.tile4.nc .........OK + Comparing dynf024.tile5.nc .........OK + Comparing dynf024.tile6.nc .........OK + Comparing RESTART/coupler.res .........OK + Comparing RESTART/fv_core.res.nc .........OK + Comparing RESTART/fv_core.res.tile1.nc .........OK + Comparing RESTART/fv_core.res.tile2.nc .........OK + Comparing RESTART/fv_core.res.tile3.nc .........OK + Comparing RESTART/fv_core.res.tile4.nc .........OK + Comparing RESTART/fv_core.res.tile5.nc .........OK + Comparing RESTART/fv_core.res.tile6.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile1.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile2.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile3.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile4.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile5.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile6.nc .........OK + Comparing RESTART/fv_tracer.res.tile1.nc .........OK + Comparing RESTART/fv_tracer.res.tile2.nc .........OK + Comparing RESTART/fv_tracer.res.tile3.nc .........OK + Comparing RESTART/fv_tracer.res.tile4.nc .........OK + Comparing RESTART/fv_tracer.res.tile5.nc .........OK + Comparing RESTART/fv_tracer.res.tile6.nc .........OK + Comparing RESTART/phy_data.tile1.nc .........OK + Comparing RESTART/phy_data.tile2.nc .........OK + Comparing RESTART/phy_data.tile3.nc .........OK + Comparing RESTART/phy_data.tile4.nc .........OK + Comparing RESTART/phy_data.tile5.nc .........OK + Comparing RESTART/phy_data.tile6.nc .........OK + Comparing RESTART/sfc_data.tile1.nc .........OK + Comparing RESTART/sfc_data.tile2.nc .........OK + Comparing RESTART/sfc_data.tile3.nc .........OK + Comparing RESTART/sfc_data.tile4.nc .........OK + Comparing RESTART/sfc_data.tile5.nc .........OK + Comparing RESTART/sfc_data.tile6.nc .........OK +Test thr fv3_ccpp_control PASS + +FAILED TESTS: +Test rst fv3_ccpp_control failed +Test rst fv3_ccpp_control failed +UNIT TEST FAILED +Tue Sep 8 17:10:09 CDT 2020 +Elapsed time: 00h:13m:51s. Have a nice day! diff --git a/tests/UnitTests_wcoss_dell_p3.log b/tests/UnitTests_wcoss_dell_p3.log new file mode 100644 index 0000000000..ac1a360f23 --- /dev/null +++ b/tests/UnitTests_wcoss_dell_p3.log @@ -0,0 +1,470 @@ + +baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_control_bit_base +working dir = /gpfs/dell2/ptmp/Minsuk.Ji/FV3_UT/ut_21236/fv3_ccpp_control_bit_base +Checking test bit_base fv3_ccpp_control results .... +Moving baseline bit_base fv3_ccpp_control files .... + mkdir -p /gpfs/dell2/stmp/Minsuk.Ji/FV3_UT/UNIT_TEST/fv3_control_bit_base/RESTART + Moving atmos_4xdaily.tile1.nc .........OK + Moving atmos_4xdaily.tile2.nc .........OK + Moving atmos_4xdaily.tile3.nc .........OK + Moving atmos_4xdaily.tile4.nc .........OK + Moving atmos_4xdaily.tile5.nc .........OK + Moving atmos_4xdaily.tile6.nc .........OK + Moving phyf000.tile1.nc .........OK + Moving phyf000.tile2.nc .........OK + Moving phyf000.tile3.nc .........OK + Moving phyf000.tile4.nc .........OK + Moving phyf000.tile5.nc .........OK + Moving phyf000.tile6.nc .........OK + Moving phyf024.tile1.nc .........OK + Moving phyf024.tile2.nc .........OK + Moving phyf024.tile3.nc .........OK + Moving phyf024.tile4.nc .........OK + Moving phyf024.tile5.nc .........OK + Moving phyf024.tile6.nc .........OK + Moving dynf000.tile1.nc .........OK + Moving dynf000.tile2.nc .........OK + Moving dynf000.tile3.nc .........OK + Moving dynf000.tile4.nc .........OK + Moving dynf000.tile5.nc .........OK + Moving dynf000.tile6.nc .........OK + Moving dynf024.tile1.nc .........OK + Moving dynf024.tile2.nc .........OK + Moving dynf024.tile3.nc .........OK + Moving dynf024.tile4.nc .........OK + Moving dynf024.tile5.nc .........OK + Moving dynf024.tile6.nc .........OK + Moving RESTART/coupler.res .........OK + Moving RESTART/fv_core.res.nc .........OK + Moving RESTART/fv_core.res.tile1.nc .........OK + Moving RESTART/fv_core.res.tile2.nc .........OK + Moving RESTART/fv_core.res.tile3.nc .........OK + Moving RESTART/fv_core.res.tile4.nc .........OK + Moving RESTART/fv_core.res.tile5.nc .........OK + Moving RESTART/fv_core.res.tile6.nc .........OK + Moving RESTART/fv_srf_wnd.res.tile1.nc .........OK + Moving RESTART/fv_srf_wnd.res.tile2.nc .........OK + Moving RESTART/fv_srf_wnd.res.tile3.nc .........OK + Moving RESTART/fv_srf_wnd.res.tile4.nc .........OK + Moving RESTART/fv_srf_wnd.res.tile5.nc .........OK + Moving RESTART/fv_srf_wnd.res.tile6.nc .........OK + Moving RESTART/fv_tracer.res.tile1.nc .........OK + Moving RESTART/fv_tracer.res.tile2.nc .........OK + Moving RESTART/fv_tracer.res.tile3.nc .........OK + Moving RESTART/fv_tracer.res.tile4.nc .........OK + Moving RESTART/fv_tracer.res.tile5.nc .........OK + Moving RESTART/fv_tracer.res.tile6.nc .........OK + Moving RESTART/phy_data.tile1.nc .........OK + Moving RESTART/phy_data.tile2.nc .........OK + Moving RESTART/phy_data.tile3.nc .........OK + Moving RESTART/phy_data.tile4.nc .........OK + Moving RESTART/phy_data.tile5.nc .........OK + Moving RESTART/phy_data.tile6.nc .........OK + Moving RESTART/sfc_data.tile1.nc .........OK + Moving RESTART/sfc_data.tile2.nc .........OK + Moving RESTART/sfc_data.tile3.nc .........OK + Moving RESTART/sfc_data.tile4.nc .........OK + Moving RESTART/sfc_data.tile5.nc .........OK + Moving RESTART/sfc_data.tile6.nc .........OK +Test bit_base fv3_ccpp_control PASS + + +baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_control_dbg_base +working dir = /gpfs/dell2/ptmp/Minsuk.Ji/FV3_UT/ut_21236/fv3_ccpp_control_dbg_base +Checking test dbg_base fv3_ccpp_control results .... +Moving baseline dbg_base fv3_ccpp_control files .... + mkdir -p /gpfs/dell2/stmp/Minsuk.Ji/FV3_UT/UNIT_TEST/fv3_control_dbg_base/RESTART + Moving atmos_4xdaily.tile1.nc .........OK + Moving atmos_4xdaily.tile2.nc .........OK + Moving atmos_4xdaily.tile3.nc .........OK + Moving atmos_4xdaily.tile4.nc .........OK + Moving atmos_4xdaily.tile5.nc .........OK + Moving atmos_4xdaily.tile6.nc .........OK + Moving phyf000.tile1.nc .........OK + Moving phyf000.tile2.nc .........OK + Moving phyf000.tile3.nc .........OK + Moving phyf000.tile4.nc .........OK + Moving phyf000.tile5.nc .........OK + Moving phyf000.tile6.nc .........OK + Moving phyf024.tile1.nc .........OK + Moving phyf024.tile2.nc .........OK + Moving phyf024.tile3.nc .........OK + Moving phyf024.tile4.nc .........OK + Moving phyf024.tile5.nc .........OK + Moving phyf024.tile6.nc .........OK + Moving dynf000.tile1.nc .........OK + Moving dynf000.tile2.nc .........OK + Moving dynf000.tile3.nc .........OK + Moving dynf000.tile4.nc .........OK + Moving dynf000.tile5.nc .........OK + Moving dynf000.tile6.nc .........OK + Moving dynf024.tile1.nc .........OK + Moving dynf024.tile2.nc .........OK + Moving dynf024.tile3.nc .........OK + Moving dynf024.tile4.nc .........OK + Moving dynf024.tile5.nc .........OK + Moving dynf024.tile6.nc .........OK + Moving RESTART/coupler.res .........OK + Moving RESTART/fv_core.res.nc .........OK + Moving RESTART/fv_core.res.tile1.nc .........OK + Moving RESTART/fv_core.res.tile2.nc .........OK + Moving RESTART/fv_core.res.tile3.nc .........OK + Moving RESTART/fv_core.res.tile4.nc .........OK + Moving RESTART/fv_core.res.tile5.nc .........OK + Moving RESTART/fv_core.res.tile6.nc .........OK + Moving RESTART/fv_srf_wnd.res.tile1.nc .........OK + Moving RESTART/fv_srf_wnd.res.tile2.nc .........OK + Moving RESTART/fv_srf_wnd.res.tile3.nc .........OK + Moving RESTART/fv_srf_wnd.res.tile4.nc .........OK + Moving RESTART/fv_srf_wnd.res.tile5.nc .........OK + Moving RESTART/fv_srf_wnd.res.tile6.nc .........OK + Moving RESTART/fv_tracer.res.tile1.nc .........OK + Moving RESTART/fv_tracer.res.tile2.nc .........OK + Moving RESTART/fv_tracer.res.tile3.nc .........OK + Moving RESTART/fv_tracer.res.tile4.nc .........OK + Moving RESTART/fv_tracer.res.tile5.nc .........OK + Moving RESTART/fv_tracer.res.tile6.nc .........OK + Moving RESTART/phy_data.tile1.nc .........OK + Moving RESTART/phy_data.tile2.nc .........OK + Moving RESTART/phy_data.tile3.nc .........OK + Moving RESTART/phy_data.tile4.nc .........OK + Moving RESTART/phy_data.tile5.nc .........OK + Moving RESTART/phy_data.tile6.nc .........OK + Moving RESTART/sfc_data.tile1.nc .........OK + Moving RESTART/sfc_data.tile2.nc .........OK + Moving RESTART/sfc_data.tile3.nc .........OK + Moving RESTART/sfc_data.tile4.nc .........OK + Moving RESTART/sfc_data.tile5.nc .........OK + Moving RESTART/sfc_data.tile6.nc .........OK +Test dbg_base fv3_ccpp_control PASS + + +baseline dir = /gpfs/dell2/stmp/Minsuk.Ji/FV3_UT/UNIT_TEST/fv3_control_std_base +working dir = /gpfs/dell2/ptmp/Minsuk.Ji/FV3_UT/ut_21236/fv3_ccpp_control_dcp +Checking test dcp fv3_ccpp_control results .... + Comparing atmos_4xdaily.tile1.nc .........OK + Comparing atmos_4xdaily.tile2.nc .........OK + Comparing atmos_4xdaily.tile3.nc .........OK + Comparing atmos_4xdaily.tile4.nc .........OK + Comparing atmos_4xdaily.tile5.nc .........OK + Comparing atmos_4xdaily.tile6.nc .........OK + Comparing phyf000.tile1.nc .........OK + Comparing phyf000.tile2.nc .........OK + Comparing phyf000.tile3.nc .........OK + Comparing phyf000.tile4.nc .........OK + Comparing phyf000.tile5.nc .........OK + Comparing phyf000.tile6.nc .........OK + Comparing phyf024.tile1.nc .........OK + Comparing phyf024.tile2.nc .........OK + Comparing phyf024.tile3.nc .........OK + Comparing phyf024.tile4.nc .........OK + Comparing phyf024.tile5.nc .........OK + Comparing phyf024.tile6.nc .........OK + Comparing dynf000.tile1.nc .........OK + Comparing dynf000.tile2.nc .........OK + Comparing dynf000.tile3.nc .........OK + Comparing dynf000.tile4.nc .........OK + Comparing dynf000.tile5.nc .........OK + Comparing dynf000.tile6.nc .........OK + Comparing dynf024.tile1.nc .........OK + Comparing dynf024.tile2.nc .........OK + Comparing dynf024.tile3.nc .........OK + Comparing dynf024.tile4.nc .........OK + Comparing dynf024.tile5.nc .........OK + Comparing dynf024.tile6.nc .........OK + Comparing RESTART/coupler.res .........OK + Comparing RESTART/fv_core.res.nc .........OK + Comparing RESTART/fv_core.res.tile1.nc .........OK + Comparing RESTART/fv_core.res.tile2.nc .........OK + Comparing RESTART/fv_core.res.tile3.nc .........OK + Comparing RESTART/fv_core.res.tile4.nc .........OK + Comparing RESTART/fv_core.res.tile5.nc .........OK + Comparing RESTART/fv_core.res.tile6.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile1.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile2.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile3.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile4.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile5.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile6.nc .........OK + Comparing RESTART/fv_tracer.res.tile1.nc .........OK + Comparing RESTART/fv_tracer.res.tile2.nc .........OK + Comparing RESTART/fv_tracer.res.tile3.nc .........OK + Comparing RESTART/fv_tracer.res.tile4.nc .........OK + Comparing RESTART/fv_tracer.res.tile5.nc .........OK + Comparing RESTART/fv_tracer.res.tile6.nc .........OK + Comparing RESTART/phy_data.tile1.nc .........OK + Comparing RESTART/phy_data.tile2.nc .........OK + Comparing RESTART/phy_data.tile3.nc .........OK + Comparing RESTART/phy_data.tile4.nc .........OK + Comparing RESTART/phy_data.tile5.nc .........OK + Comparing RESTART/phy_data.tile6.nc .........OK + Comparing RESTART/sfc_data.tile1.nc .........OK + Comparing RESTART/sfc_data.tile2.nc .........OK + Comparing RESTART/sfc_data.tile3.nc .........OK + Comparing RESTART/sfc_data.tile4.nc .........OK + Comparing RESTART/sfc_data.tile5.nc .........OK + Comparing RESTART/sfc_data.tile6.nc .........OK +Test dcp fv3_ccpp_control PASS + + +baseline dir = /gpfs/dell2/stmp/Minsuk.Ji/FV3_UT/UNIT_TEST/fv3_control_std_base +working dir = /gpfs/dell2/ptmp/Minsuk.Ji/FV3_UT/ut_21236/fv3_ccpp_control_mpi +Checking test mpi fv3_ccpp_control results .... + Comparing atmos_4xdaily.tile1.nc .........OK + Comparing atmos_4xdaily.tile2.nc .........OK + Comparing atmos_4xdaily.tile3.nc .........OK + Comparing atmos_4xdaily.tile4.nc .........OK + Comparing atmos_4xdaily.tile5.nc .........OK + Comparing atmos_4xdaily.tile6.nc .........OK + Comparing phyf000.tile1.nc .........OK + Comparing phyf000.tile2.nc .........OK + Comparing phyf000.tile3.nc .........OK + Comparing phyf000.tile4.nc .........OK + Comparing phyf000.tile5.nc .........OK + Comparing phyf000.tile6.nc .........OK + Comparing phyf024.tile1.nc .........OK + Comparing phyf024.tile2.nc .........OK + Comparing phyf024.tile3.nc .........OK + Comparing phyf024.tile4.nc .........OK + Comparing phyf024.tile5.nc .........OK + Comparing phyf024.tile6.nc .........OK + Comparing dynf000.tile1.nc .........OK + Comparing dynf000.tile2.nc .........OK + Comparing dynf000.tile3.nc .........OK + Comparing dynf000.tile4.nc .........OK + Comparing dynf000.tile5.nc .........OK + Comparing dynf000.tile6.nc .........OK + Comparing dynf024.tile1.nc .........OK + Comparing dynf024.tile2.nc .........OK + Comparing dynf024.tile3.nc .........OK + Comparing dynf024.tile4.nc .........OK + Comparing dynf024.tile5.nc .........OK + Comparing dynf024.tile6.nc .........OK + Comparing RESTART/coupler.res .........OK + Comparing RESTART/fv_core.res.nc .........OK + Comparing RESTART/fv_core.res.tile1.nc .........OK + Comparing RESTART/fv_core.res.tile2.nc .........OK + Comparing RESTART/fv_core.res.tile3.nc .........OK + Comparing RESTART/fv_core.res.tile4.nc .........OK + Comparing RESTART/fv_core.res.tile5.nc .........OK + Comparing RESTART/fv_core.res.tile6.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile1.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile2.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile3.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile4.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile5.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile6.nc .........OK + Comparing RESTART/fv_tracer.res.tile1.nc .........OK + Comparing RESTART/fv_tracer.res.tile2.nc .........OK + Comparing RESTART/fv_tracer.res.tile3.nc .........OK + Comparing RESTART/fv_tracer.res.tile4.nc .........OK + Comparing RESTART/fv_tracer.res.tile5.nc .........OK + Comparing RESTART/fv_tracer.res.tile6.nc .........OK + Comparing RESTART/phy_data.tile1.nc .........OK + Comparing RESTART/phy_data.tile2.nc .........OK + Comparing RESTART/phy_data.tile3.nc .........OK + Comparing RESTART/phy_data.tile4.nc .........OK + Comparing RESTART/phy_data.tile5.nc .........OK + Comparing RESTART/phy_data.tile6.nc .........OK + Comparing RESTART/sfc_data.tile1.nc .........OK + Comparing RESTART/sfc_data.tile2.nc .........OK + Comparing RESTART/sfc_data.tile3.nc .........OK + Comparing RESTART/sfc_data.tile4.nc .........OK + Comparing RESTART/sfc_data.tile5.nc .........OK + Comparing RESTART/sfc_data.tile6.nc .........OK +Test mpi fv3_ccpp_control PASS + + +baseline dir = /gpfs/dell2/stmp/Minsuk.Ji/FV3_UT/UNIT_TEST/fv3_control_std_base +working dir = /gpfs/dell2/ptmp/Minsuk.Ji/FV3_UT/ut_21236/fv3_ccpp_control_rst +Checking test rst fv3_ccpp_control results .... + Comparing phyf024.tile1.nc ............NOT OK + Comparing phyf024.tile2.nc ............NOT OK + Comparing phyf024.tile3.nc ............NOT OK + Comparing phyf024.tile4.nc ............NOT OK + Comparing phyf024.tile5.nc ............NOT OK + Comparing phyf024.tile6.nc ............NOT OK + Comparing dynf024.tile1.nc ............NOT OK + Comparing dynf024.tile2.nc ............NOT OK + Comparing dynf024.tile3.nc ............NOT OK + Comparing dynf024.tile4.nc ............NOT OK + Comparing dynf024.tile5.nc ............NOT OK + Comparing dynf024.tile6.nc ............NOT OK + Comparing RESTART/coupler.res .........OK + Comparing RESTART/fv_core.res.nc .........OK + Comparing RESTART/fv_core.res.tile1.nc ............NOT OK + Comparing RESTART/fv_core.res.tile2.nc ............NOT OK + Comparing RESTART/fv_core.res.tile3.nc ............NOT OK + Comparing RESTART/fv_core.res.tile4.nc ............NOT OK + Comparing RESTART/fv_core.res.tile5.nc ............NOT OK + Comparing RESTART/fv_core.res.tile6.nc ............NOT OK + Comparing RESTART/fv_srf_wnd.res.tile1.nc ............NOT OK + Comparing RESTART/fv_srf_wnd.res.tile2.nc ............NOT OK + Comparing RESTART/fv_srf_wnd.res.tile3.nc ............NOT OK + Comparing RESTART/fv_srf_wnd.res.tile4.nc ............NOT OK + Comparing RESTART/fv_srf_wnd.res.tile5.nc ............NOT OK + Comparing RESTART/fv_srf_wnd.res.tile6.nc ............NOT OK + Comparing RESTART/fv_tracer.res.tile1.nc ............NOT OK + Comparing RESTART/fv_tracer.res.tile2.nc ............NOT OK + Comparing RESTART/fv_tracer.res.tile3.nc ............NOT OK + Comparing RESTART/fv_tracer.res.tile4.nc ............NOT OK + Comparing RESTART/fv_tracer.res.tile5.nc ............NOT OK + Comparing RESTART/fv_tracer.res.tile6.nc ............NOT OK + Comparing RESTART/phy_data.tile1.nc ............NOT OK + Comparing RESTART/phy_data.tile2.nc ............NOT OK + Comparing RESTART/phy_data.tile3.nc ............NOT OK + Comparing RESTART/phy_data.tile4.nc ............NOT OK + Comparing RESTART/phy_data.tile5.nc ............NOT OK + Comparing RESTART/phy_data.tile6.nc ............NOT OK + Comparing RESTART/sfc_data.tile1.nc ............NOT OK + Comparing RESTART/sfc_data.tile2.nc ............NOT OK + Comparing RESTART/sfc_data.tile3.nc ............NOT OK + Comparing RESTART/sfc_data.tile4.nc ............NOT OK + Comparing RESTART/sfc_data.tile5.nc ............NOT OK + Comparing RESTART/sfc_data.tile6.nc ............NOT OK +Test rst fv3_ccpp_control FAIL + + +baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_control_std_base +working dir = /gpfs/dell2/ptmp/Minsuk.Ji/FV3_UT/ut_21236/fv3_ccpp_control_std_base +Checking test std_base fv3_ccpp_control results .... +Moving baseline std_base fv3_ccpp_control files .... + mkdir -p /gpfs/dell2/stmp/Minsuk.Ji/FV3_UT/UNIT_TEST/fv3_control_std_base/RESTART + Moving atmos_4xdaily.tile1.nc .........OK + Moving atmos_4xdaily.tile2.nc .........OK + Moving atmos_4xdaily.tile3.nc .........OK + Moving atmos_4xdaily.tile4.nc .........OK + Moving atmos_4xdaily.tile5.nc .........OK + Moving atmos_4xdaily.tile6.nc .........OK + Moving phyf000.tile1.nc .........OK + Moving phyf000.tile2.nc .........OK + Moving phyf000.tile3.nc .........OK + Moving phyf000.tile4.nc .........OK + Moving phyf000.tile5.nc .........OK + Moving phyf000.tile6.nc .........OK + Moving phyf024.tile1.nc .........OK + Moving phyf024.tile2.nc .........OK + Moving phyf024.tile3.nc .........OK + Moving phyf024.tile4.nc .........OK + Moving phyf024.tile5.nc .........OK + Moving phyf024.tile6.nc .........OK + Moving dynf000.tile1.nc .........OK + Moving dynf000.tile2.nc .........OK + Moving dynf000.tile3.nc .........OK + Moving dynf000.tile4.nc .........OK + Moving dynf000.tile5.nc .........OK + Moving dynf000.tile6.nc .........OK + Moving dynf024.tile1.nc .........OK + Moving dynf024.tile2.nc .........OK + Moving dynf024.tile3.nc .........OK + Moving dynf024.tile4.nc .........OK + Moving dynf024.tile5.nc .........OK + Moving dynf024.tile6.nc .........OK + Moving RESTART/coupler.res .........OK + Moving RESTART/fv_core.res.nc .........OK + Moving RESTART/fv_core.res.tile1.nc .........OK + Moving RESTART/fv_core.res.tile2.nc .........OK + Moving RESTART/fv_core.res.tile3.nc .........OK + Moving RESTART/fv_core.res.tile4.nc .........OK + Moving RESTART/fv_core.res.tile5.nc .........OK + Moving RESTART/fv_core.res.tile6.nc .........OK + Moving RESTART/fv_srf_wnd.res.tile1.nc .........OK + Moving RESTART/fv_srf_wnd.res.tile2.nc .........OK + Moving RESTART/fv_srf_wnd.res.tile3.nc .........OK + Moving RESTART/fv_srf_wnd.res.tile4.nc .........OK + Moving RESTART/fv_srf_wnd.res.tile5.nc .........OK + Moving RESTART/fv_srf_wnd.res.tile6.nc .........OK + Moving RESTART/fv_tracer.res.tile1.nc .........OK + Moving RESTART/fv_tracer.res.tile2.nc .........OK + Moving RESTART/fv_tracer.res.tile3.nc .........OK + Moving RESTART/fv_tracer.res.tile4.nc .........OK + Moving RESTART/fv_tracer.res.tile5.nc .........OK + Moving RESTART/fv_tracer.res.tile6.nc .........OK + Moving RESTART/phy_data.tile1.nc .........OK + Moving RESTART/phy_data.tile2.nc .........OK + Moving RESTART/phy_data.tile3.nc .........OK + Moving RESTART/phy_data.tile4.nc .........OK + Moving RESTART/phy_data.tile5.nc .........OK + Moving RESTART/phy_data.tile6.nc .........OK + Moving RESTART/sfc_data.tile1.nc .........OK + Moving RESTART/sfc_data.tile2.nc .........OK + Moving RESTART/sfc_data.tile3.nc .........OK + Moving RESTART/sfc_data.tile4.nc .........OK + Moving RESTART/sfc_data.tile5.nc .........OK + Moving RESTART/sfc_data.tile6.nc .........OK +Test std_base fv3_ccpp_control PASS + + +baseline dir = /gpfs/dell2/stmp/Minsuk.Ji/FV3_UT/UNIT_TEST/fv3_control_std_base +working dir = /gpfs/dell2/ptmp/Minsuk.Ji/FV3_UT/ut_21236/fv3_ccpp_control_thr +Checking test thr fv3_ccpp_control results .... + Comparing atmos_4xdaily.tile1.nc .........OK + Comparing atmos_4xdaily.tile2.nc .........OK + Comparing atmos_4xdaily.tile3.nc .........OK + Comparing atmos_4xdaily.tile4.nc .........OK + Comparing atmos_4xdaily.tile5.nc .........OK + Comparing atmos_4xdaily.tile6.nc .........OK + Comparing phyf000.tile1.nc .........OK + Comparing phyf000.tile2.nc .........OK + Comparing phyf000.tile3.nc .........OK + Comparing phyf000.tile4.nc .........OK + Comparing phyf000.tile5.nc .........OK + Comparing phyf000.tile6.nc .........OK + Comparing phyf024.tile1.nc .........OK + Comparing phyf024.tile2.nc .........OK + Comparing phyf024.tile3.nc .........OK + Comparing phyf024.tile4.nc .........OK + Comparing phyf024.tile5.nc .........OK + Comparing phyf024.tile6.nc .........OK + Comparing dynf000.tile1.nc .........OK + Comparing dynf000.tile2.nc .........OK + Comparing dynf000.tile3.nc .........OK + Comparing dynf000.tile4.nc .........OK + Comparing dynf000.tile5.nc .........OK + Comparing dynf000.tile6.nc .........OK + Comparing dynf024.tile1.nc .........OK + Comparing dynf024.tile2.nc .........OK + Comparing dynf024.tile3.nc .........OK + Comparing dynf024.tile4.nc .........OK + Comparing dynf024.tile5.nc .........OK + Comparing dynf024.tile6.nc .........OK + Comparing RESTART/coupler.res .........OK + Comparing RESTART/fv_core.res.nc .........OK + Comparing RESTART/fv_core.res.tile1.nc .........OK + Comparing RESTART/fv_core.res.tile2.nc .........OK + Comparing RESTART/fv_core.res.tile3.nc .........OK + Comparing RESTART/fv_core.res.tile4.nc .........OK + Comparing RESTART/fv_core.res.tile5.nc .........OK + Comparing RESTART/fv_core.res.tile6.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile1.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile2.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile3.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile4.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile5.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile6.nc .........OK + Comparing RESTART/fv_tracer.res.tile1.nc .........OK + Comparing RESTART/fv_tracer.res.tile2.nc .........OK + Comparing RESTART/fv_tracer.res.tile3.nc .........OK + Comparing RESTART/fv_tracer.res.tile4.nc .........OK + Comparing RESTART/fv_tracer.res.tile5.nc .........OK + Comparing RESTART/fv_tracer.res.tile6.nc .........OK + Comparing RESTART/phy_data.tile1.nc .........OK + Comparing RESTART/phy_data.tile2.nc .........OK + Comparing RESTART/phy_data.tile3.nc .........OK + Comparing RESTART/phy_data.tile4.nc .........OK + Comparing RESTART/phy_data.tile5.nc .........OK + Comparing RESTART/phy_data.tile6.nc .........OK + Comparing RESTART/sfc_data.tile1.nc .........OK + Comparing RESTART/sfc_data.tile2.nc .........OK + Comparing RESTART/sfc_data.tile3.nc .........OK + Comparing RESTART/sfc_data.tile4.nc .........OK + Comparing RESTART/sfc_data.tile5.nc .........OK + Comparing RESTART/sfc_data.tile6.nc .........OK +Test thr fv3_ccpp_control PASS + +FAILED TESTS: +Test rst fv3_ccpp_control failed +Test rst fv3_ccpp_control failed +UNIT TEST FAILED +Tue Sep 8 21:39:25 UTC 2020 +Elapsed time: 00h:32m:00s. Have a nice day! diff --git a/tests/ci/Dockerfile b/tests/ci/Dockerfile new file mode 100644 index 0000000000..b4a5352efa --- /dev/null +++ b/tests/ci/Dockerfile @@ -0,0 +1,20 @@ +From minsukjinoaa/ci-test-base:ubuntu20.04-v3 +#From minsukjinoaa/ci-test-base:centos7-v3 + +ENV HOME=/home/tester +COPY --chown=tester:tester . $HOME/ufs-weather-model + +USER tester +ENV USER=tester +ARG test_name +ARG build_case +ENV test_name=$test_name +ENV build_case=$build_case +ENV test_case= +ENV CI_TEST=true +ENV NEMS_COMPILER=gnu +ENV NEMS_MACHINE=linux.gnu + +WORKDIR $HOME/ufs-weather-model/tests +RUN ./utest -n $test_name -c $build_case -z +CMD ./utest -n $test_name -c $test_case -x diff --git a/tests/ci/ci.sh b/tests/ci/ci.sh new file mode 100755 index 0000000000..d332078e5f --- /dev/null +++ b/tests/ci/ci.sh @@ -0,0 +1,88 @@ +#!/bin/bash +set -eu + +check_memory_usage() { + dirName=/sys/fs/cgroup/memory/docker/$1 + # /sys/fs/cgroup/memory/actions_job/${containerID} + set +x + while [ -d $dirName ] ; do + awk '/(^cache |^rss |^shmem )/' $dirName/memory.stat | cut -f2 -d' ' | paste -s -d, + sleep 1 + done + set -x +} + +usage_and_exit() { + echo + echo "Note: main purpose of this script is to interface between CI automation and utest script" + echo "and therefore, direct invocation via CLI may result in unexpected behavior" + echo + echo "Usage: $0 -b | -r " + echo " -b specify cases to build: comma-separated list of any combination of std,big,dbg" + echo " -r specify tests to run: comma-separated list of any combination of std,thr,mpi,dcp,rst,bit,dbg" + echo + exit 2 +} + +IMG_NAME=$(sed -n 3p ci.test) +BUILD="false" +RUN="false" +TEST_NAME="" +BUILD_CASE="" +TEST_CASE="" + +while getopts :b:r:n: opt; do + case $opt in + b) + BUILD="true" + BUILD_CASE=$OPTARG + ;; + r) + RUN="true" + TEST_CASE=$OPTARG + ;; + n) + TEST_NAME=$OPTARG + ;; + esac +done + +# Read in TEST_NAME if not passed on +TEST_NAME=${TEST_NAME:-$(sed -n 1p ci.test)} +echo "test name is ${TEST_NAME}" + +if [ $BUILD = "true" ] && [ $RUN = "true" ]; then + echo "Specify either build (-b) or run (-r) option, not both" + usage_and_exit +fi + +if [ $BUILD = "false" ] && [ $RUN = "false" ]; then + echo "Specify either build (-b) or run (-r) option" + usage_and_exit +fi + +if [ $BUILD = "true" ]; then + + sed -i -e '/affinity.c/d' ../../CMakeLists.txt + + sudo docker build --build-arg test_name=$TEST_NAME \ + --build-arg build_case=$BUILD_CASE \ + --no-cache \ + --squash --compress \ + -f Dockerfile -t ${IMG_NAME} ../.. + exit $? + +elif [ $RUN == "true" ]; then + + sudo docker run -d --rm -v DataVolume:/tmp minsukjinoaa/fv3-input-data:develop-20200713 + sudo docker run -d -e test_case=${TEST_CASE} -v DataVolume:/home/tester/data/NEMSfv3gfs/develop-20200713 --name my-container ${IMG_NAME} + + echo 'cache,rss,shmem' >memory_stat + sleep 3 + containerID=$(sudo docker ps -q --no-trunc) + check_memory_usage $containerID >>memory_stat & + + sudo docker logs -f $containerID + exit $(sudo docker inspect $containerID --format='{{.State.ExitCode}}') + +fi diff --git a/tests/ci/ci.test b/tests/ci/ci.test new file mode 100644 index 0000000000..a5fa283cbb --- /dev/null +++ b/tests/ci/ci.test @@ -0,0 +1,3 @@ +fv3_ccpp_control +bit +ci-test-weather diff --git a/tests/ci/json_helper.py b/tests/ci/json_helper.py new file mode 100755 index 0000000000..cc4fd4f32c --- /dev/null +++ b/tests/ci/json_helper.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +# The following env variables are assumed available and valid: +# GITHUB_ACTOR, GITHUB_RUN_ID, TRIGGER_ID, TRIGGER_BR +import os +import re +import sys +import json + +def check_skip(data): + msg = data["head_commit"]["message"] + if re.search("skip-ci", msg): + return "yes" + else: + return "no" + +def cancel_workflow(data): + wfs=[x["id"] for x in data if x["head_repository"] is not None and + re.search(os.environ["GITHUB_ACTOR"], x["head_repository"]["owner"]["login"]) and + x["id"]!=int(os.environ["GITHUB_RUN_ID"]) and + x["id"]!=int(os.environ["TRIGGER_ID"]) and + x["head_branch"]==os.environ["TRIGGER_BR"] and + (x["status"]=="queued" or x["status"]=="in_progress")] + + return wfs + +def main(): + + if sys.argv[1]=="check_skip": + data = json.load(sys.stdin)["workflow_run"] + ans = check_skip(data) + print(ans) + elif sys.argv[1]=="get_trigger_id": + print(json.load(sys.stdin)["workflow_run"]["id"]) + elif sys.argv[1]=="get_trigger_br": + print(json.load(sys.stdin)["workflow_run"]["head_branch"]) + elif sys.argv[1]=="cancel_workflow": + data = json.load(sys.stdin)["workflow_runs"] + wfs = cancel_workflow(data) + if len(wfs)==0: + print("") + else: + print(*wfs) + else: + print("ERROR") + +if __name__ == "__main__": main() diff --git a/tests/ci/parse.sh b/tests/ci/parse.sh new file mode 100755 index 0000000000..326da18ece --- /dev/null +++ b/tests/ci/parse.sh @@ -0,0 +1,35 @@ +#!/bin/bash +set -eu + +name_=$(sed -n 1p ci.test) +case_=$(sed -n 2p ci.test) +img_=$(sed -n 3p ci.test) + +imd1_='' +[[ $case_ =~ thr || $case_ =~ mpi || $case_ =~ dcp || $case_ =~ rst ]] && imd1_+='std' +[[ $case_ =~ bit ]] && imd1_+=' bit' +[[ $case_ =~ dbg ]] && imd1_+=' dbg' +imd1_=$(echo $imd1_ | sed -e 's/^ *//' -e 's/ *$//') + +bld_='{"bld_set":[' +for i in $imd1_; do bld_+="\"$i\","; done +bld_=$(echo $bld_ | sed -e 's/,$//') +bld_+=']}' + +imd2_=() +test_='{"test_set":[' +for i in $case_; do + test_+="\"$i\"," + [[ $i =~ thr || $i =~ mpi || $i =~ dcp || $i =~ rst ]] && imd2_+=( std ) + [[ $i =~ bit ]] && imd2_+=( bit ) + [[ $i =~ dbg ]] && imd2_+=( dbg ) +done + +test_=$(echo $test_ | sed -e 's/,$//') +test_+='],"include":[' +j=0 +for i in $case_; do test_+="{\"test_set\":\"$i\",\"artifact\":\"${imd2_[$j]}\"},"; j=$((j+1)); done +test_=$(echo $test_ | sed -e 's/,$//') +test_+=']}' + +echo $name_ $bld_ $test_ $img_ diff --git a/tests/default_vars.sh b/tests/default_vars.sh index 9194887c2e..93c7e06ae7 100755 --- a/tests/default_vars.sh +++ b/tests/default_vars.sh @@ -35,6 +35,17 @@ elif [[ $MACHINE_ID = hera.* ]]; then TASKS_stretch=48 ; TPN_stretch=12 ; INPES_stretch=2 ; JNPES_stretch=4 TASKS_strnest=96 ; TPN_strnest=12 ; INPES_strnest=2 ; JNPES_strnest=4 +elif [[ $MACHINE_ID = linux.* ]]; then + + if [[ $CI_TEST = true ]]; then + TASKS_dflt=12 ; TPN_dflt=16 ; INPES_dflt=1 ; JNPES_dflt=1 + else + TASKS_dflt=150 ; TPN_dflt=40 ; INPES_dflt=3 ; JNPES_dflt=8 + fi + TASKS_thrd=84 ; TPN_thrd=20 ; INPES_thrd=3 ; JNPES_thrd=4 + TASKS_stretch=48 ; TPN_stretch=12 ; INPES_stretch=2 ; JNPES_stretch=4 + TASKS_strnest=96 ; TPN_strnest=12 ; INPES_strnest=2 ; JNPES_strnest=4 + elif [[ $MACHINE_ID = jet.* ]]; then TASKS_dflt=150 ; TPN_dflt=24 ; INPES_dflt=3 ; JNPES_dflt=8 diff --git a/tests/rt.conf b/tests/rt.conf index 42608beae6..006789bec4 100644 --- a/tests/rt.conf +++ b/tests/rt.conf @@ -147,7 +147,7 @@ RUN | fv3_ccpp_gfs_v16beta_flake # Exercise compilation without specifying suites (i.e. compile all suites) in DEBUG mode (faster than in PROD mode) COMPILE | CCPP=Y DEBUG=Y | standard | hera.intel | fv3 | -COMPILE | CCPP=Y DEBUG=Y | standard | orion.intel | fv3 | +COMPILE | CCPP=Y SUITES=FV3_GFS_v15p2,FV3_GFS_v16beta,FV3_GFS_v15p2_RRTMGP,FV3_GFS_v16beta_RRTMGP DEBUG=Y | standard | orion.intel | fv3 | COMPILE | CCPP=Y DEBUG=Y | standard | cheyenne.intel | fv3 | RUN | fv3_ccpp_gfs_v15p2_debug | standard | hera.intel | fv3 | RUN | fv3_ccpp_gfs_v15p2_debug | standard | orion.intel | fv3 | diff --git a/tests/rt_utils.sh b/tests/rt_utils.sh index acb5c6cfc0..944792a484 100755 --- a/tests/rt_utils.sh +++ b/tests/rt_utils.sh @@ -323,13 +323,14 @@ check_results() { for i in ${LIST_FILES} ; do printf %s " Moving " $i " ....." printf %s " Moving " $i " ....." >> ${REGRESSIONTEST_LOG} + printf %s " Moving " $i " ....." if [[ -f ${RUNDIR}/$i ]] ; then cp ${RUNDIR}/${i} ${NEW_BASELINE}/${CNTL_DIR}/${i} - echo ".... OK" - echo ".... OK" >> ${REGRESSIONTEST_LOG} + echo "....OK" >>${REGRESSIONTEST_LOG} + echo "....OK" else - echo ".... missing " ${RUNDIR}/$i - echo ".... missing " ${RUNDIR}/$i >> ${REGRESSIONTEST_LOG} + echo "....NOT OK. Missing " ${RUNDIR}/$i >>${REGRESSIONTEST_LOG} + echo "....NOT OK. Missing " ${RUNDIR}/$i test_status='FAIL' fi done diff --git a/tests/run_test.sh b/tests/run_test.sh index e7115f8b5c..9ce0db15aa 100755 --- a/tests/run_test.sh +++ b/tests/run_test.sh @@ -130,11 +130,20 @@ atparse < ${PATHTR}/parm/${NEMS_CONFIGURE:-nems.configure} > nems.configure # Submit test job ################################################################################ -if [[ $ROCOTO = 'false' ]]; then - submit_and_wait job_card +if [[ $SCHEDULER = 'none' ]]; then + + ulimit -s unlimited + mpiexec -n ${TASKS} ./fv3.exe >out 2> >(tee err >&3) + else - chmod u+x job_card - ./job_card + + if [[ $ROCOTO = 'false' ]]; then + submit_and_wait job_card + else + chmod u+x job_card + ./job_card + fi + fi check_results diff --git a/tests/utest b/tests/utest index 769f0a1f08..f238151a04 100755 --- a/tests/utest +++ b/tests/utest @@ -1,5 +1,6 @@ #!/bin/bash set -eu +exec 3>&2 SECONDS=0 hostname @@ -12,34 +13,30 @@ error() { usage() { #set +x echo - echo "Usage: $program -n [ -c | -r ] [-e] [-k] [-h]" + echo "Usage: $program -n [ -c ] [-b] [-d] [-e] [-k] [-h] [-x] [-z]" echo echo " -n specify " echo - echo " -c create new baseline results. is" - echo " either 'all' or any combination of 'std','32bit','debug'" + echo " -c specify " + echo " defaults to all test-cases: thr,mpi,dcp,rst,bit,dbg" + echo " comma-separated list of any combination of std,thr,mpi,dcp,rst,bit,dbg" echo - echo " -r run unit tests. is either 'all' or any combination" - echo " of 'std','thread','mpi','decomp','restart','32bit','debug'" - echo - echo " -e use ecflow workflow manager" + echo " -b test reproducibility for bit; compare against baseline" + echo " -d test reproducibility for dbg; compare against baseline" + echo " -s test reproducibility for std; compare against baseline" + echo " -e use ecFlow workflow manager" echo " -k keep run directory" echo " -h display this help and exit" + echo " -x skip compile" + echo " -z skip run" echo echo " Examples" echo - echo " To create new baselines and run unit tests:" - echo " 'utest -n fv3_thompson' creates and runs all cases for fv3_thompson" - echo - echo " To create new baselines only:" - echo " 'utest -n fv3_control -c all' creates std, 32bit, debug for fv3_control" - echo " 'utest -n fv3_iau -c std,debug' creates std, debug for fv3_iau" - echo " 'utest -n fv3_gfdlmprad_gws -c 32bit' creates 32bit for fv3_gfdlmprad_gws" - echo - echo " To run unit tests only (baselines should exist to compare results against):" - echo " 'utest -n fv3_cpt -r all' runs std,thread,mpi,decomp,restart,32bit,debug for fv3_cpt" - echo " 'utest -n fv3_control -r thread,decomp' runs thread,decomp for fv3_control" - echo " 'utest -n fv3_stochy -r restart' runs restart for fv3_stochy" + echo " 'utest -n fv3_ccpp_control' tests all for fv3_ccpp_control" + echo " 'utest -n fv3_ccpp_control -c rst' tests restart for fv3_ccpp_control" + echo " 'utest -n fv3_ccpp_gfdlmp -c mpi,thr' tests mpi and threading for fv3_ccpp_gfdlmp" + echo " 'utest -n fv3_ccpp_gfdlmp -c bit' tests bit for fv3_ccpp_gfdlmp" + echo " 'utest -n fv3_ccpp_satmedmf -c dcp,dbg,rst' tests decomposition, debug, restart for fv3_ccpp_satmedmf" echo #set -x } @@ -52,37 +49,115 @@ usage_and_exit() { cleanup() { rm -rf ${lockdir} [[ ${ECFLOW:-false} == true ]] && ecflow_stop - trap 0 - exit + exit $1 +} + +build_utests() { + [[ -f $build_file ]] || error "$build_file does not exist" + + rm -f fv3_*.exe modules.fv3_* $compile_log + + model_found=false + base_opt= + while IFS="|" read model compile_opt; do + model=$(echo $model | sed -e 's/^ *//' -e 's/ *$//') + compile_opt=$(echo $compile_opt | sed -e 's/^ *//' -e 's/ *$//') + if [[ $model == $TEST_NAME ]]; then + base_opt=$compile_opt + model_found=true + break + fi + done <$build_file + + if [[ $model_found == false ]]; then + error "build options for $TEST_NAME not found. please edit utest.bld" + fi + + for name in $compile_case; do + case $name in + std) + NEMS_VER=$base_opt + ;; + bit) + if [[ $base_opt =~ "32BIT=Y" ]]; then + NEMS_VER=$(echo $base_opt | sed -e 's/32BIT=Y/32BIT=N/') + elif [[ ${base_opt} =~ "32BIT=N" ]]; then + NEMS_VER=$(echo $base_opt | sed -e 's/32BIT=N/32BIT=Y/') + else + NEMS_VER="$base_opt 32BIT=Y" + fi + ;; + dbg) + NEMS_VER="$base_opt 32BIT=Y DEBUG=Y" + ;; + esac + NEMS_VER=$(echo $NEMS_VER | sed -e 's/^ *//' -e 's/ *$//') + + cat <<-EOF > ${RUNDIR_ROOT}/compile_${name}.env + export MACHINE_ID=${MACHINE_ID} + export PATHRT=${PATHRT} + export PATHTR=${PATHTR} + export SCHEDULER=${SCHEDULER} + export ACCNR=${ACCNR} + export QUEUE=${COMPILE_QUEUE} + export PARTITION=${PARTITION} + export ROCOTO=${ROCOTO} + export ECFLOW=${ECFLOW} + export REGRESSIONTEST_LOG=${REGRESSIONTEST_LOG} + export LOG_DIR=${LOG_DIR} + EOF + + if [[ $ECFLOW == true ]]; then + COMPILE_NR=$name + APP='' + ecflow_create_compile_task + else + echo "compiling $name with compile option $NEMS_VER" + ./compile_cmake.sh $PATHTR $MACHINE_ID "${NEMS_VER}" $name >${LOG_DIR}/compile_${TEST_NAME}_$name.log 2>&1 + echo "done compiling $name" + fi + done } run_utests() { - for rc in $ut_run_cases; do - # Load namelist default and override values + for rc in $run_case; do + # load default variables and override as necessary source default_vars.sh source ${PATHRT}/tests/$TEST_NAME - # If TEST_NAME specifies WARM_START true, error and exit + # if TEST_NAME specifies WARM_START true, error and exit if [[ ${WARM_START} == .T. ]]; then - error "test-name cannot be a restart run; i.e., WARM_START=.T." + error "test-name cannot be a restart run (i.e. WARM_START=.T.)" fi comp_nm=std - if [[ ${create_flag} == true ]]; then - RT_SUFFIX="_${rc}_base" - else - RT_SUFFIX="_${rc}" - fi + RT_SUFFIX="_${rc}" case $rc in - std) + std_base) + CREATE_BASELINE=true + RTPWD=$baseline_location + BL_SUFFIX=_std_base cat <<-EOF > ${RUNDIR_ROOT}/unit_test${RT_SUFFIX}.env export UNIT_TEST=true + export NEMS_COMPILER=${NEMS_COMPILER} export RESTART_INTERVAL=$(( FHMAX/2 )) EOF ;; - thread) + std) + CREATE_BASELINE=false + RTPWD=$NEW_BASELINE + BL_SUFFIX=_std_base + cat <<-EOF > ${RUNDIR_ROOT}/unit_test${RT_SUFFIX}.env + export UNIT_TEST=true + export NEMS_COMPILER=${NEMS_COMPILER} + EOF + ;; + thr) + CREATE_BASELINE=false + RTPWD=$NEW_BASELINE + BL_SUFFIX=_std_base THRD=2 # INPES is sometimes odd, so use JNPES. Make sure JNPES is divisible by THRD JNPES=$(( JNPES/THRD )) @@ -91,6 +166,7 @@ run_utests() { NODES=$(( TASKS/TPN + 1 )) cat <<-EOF > ${RUNDIR_ROOT}/unit_test${RT_SUFFIX}.env export UNIT_TEST=true + export NEMS_COMPILER=${NEMS_COMPILER} export THRD=${THRD} export JNPES=${JNPES} export TASKS=${TASKS} @@ -99,6 +175,9 @@ run_utests() { EOF ;; mpi) + CREATE_BASELINE=false + RTPWD=$NEW_BASELINE + BL_SUFFIX=_std_base JNPES=$(( JNPES/2 )) WRITE_GROUP=2 WRTTASK_PER_GROUP=12 @@ -106,6 +185,7 @@ run_utests() { NODES=$(( TASKS/TPN + 1 )) cat <<-EOF > ${RUNDIR_ROOT}/unit_test${RT_SUFFIX}.env export UNIT_TEST=true + export NEMS_COMPILER=${NEMS_COMPILER} export JNPES=${JNPES} export WRITE_GROUP=${WRITE_GROUP} export WRTTASK_PER_GROUP=${WRTTASK_PER_GROUP} @@ -113,17 +193,24 @@ run_utests() { export NODES=${NODES} EOF ;; - decomp) + dcp) + CREATE_BASELINE=false + RTPWD=$NEW_BASELINE + BL_SUFFIX=_std_base temp=$INPES INPES=$JNPES JNPES=$temp cat <<-EOF >${RUNDIR_ROOT}/unit_test${RT_SUFFIX}.env export UNIT_TEST=true + export NEMS_COMPILER=${NEMS_COMPILER} export INPES=${INPES} export JNPES=${JNPES} EOF ;; - restart) # this is not going to work for regional model + rst) # this is not going to work for regional model + CREATE_BASELINE=false + RTPWD=$NEW_BASELINE + BL_SUFFIX=_std_base # Set up date and time of restart files for restart run RESTART_FILE_PREFIX="${SYEAR}${SMONTH}${SDAY}.$(printf "%02d" $(( SHOUR + FHMAX/2 )))0000" export RESTART_FILE_PREFIX @@ -171,6 +258,7 @@ run_utests() { cat <<-EOF >${RUNDIR_ROOT}/unit_test${RT_SUFFIX}.env export UNIT_TEST=true + export NEMS_COMPILER=${NEMS_COMPILER} export RESTART_FILE_PREFIX=${RESTART_FILE_PREFIX} export WARM_START=${WARM_START} export NGGPS_IC=${NGGPS_IC} @@ -182,65 +270,88 @@ run_utests() { export LIST_FILES="${LIST_FILES}" EOF ;; - 32bit) - comp_nm=$rc + bit_base) + CREATE_BASELINE=true + RTPWD=$baseline_location + BL_SUFFIX=_bit_base + comp_nm=bit cat <<-EOF >${RUNDIR_ROOT}/unit_test${RT_SUFFIX}.env export UNIT_TEST=true + export NEMS_COMPILER=${NEMS_COMPILER} EOF ;; - debug) - comp_nm=$rc + bit) + CREATE_BASELINE=false + RTPWD=$NEW_BASELINE + BL_SUFFIX=_bit_base + comp_nm=bit + cat <<-EOF >${RUNDIR_ROOT}/unit_test${RT_SUFFIX}.env + export UNIT_TEST=true + export NEMS_COMPILER=${NEMS_COMPILER} + EOF + ;; + dbg_base) + CREATE_BASELINE=true + RTPWD=$baseline_location + BL_SUFFIX=_dbg_base + comp_nm=dbg WLCLK=30 cat <<-EOF >${RUNDIR_ROOT}/unit_test${RT_SUFFIX}.env export UNIT_TEST=true + export NEMS_COMPILER=${NEMS_COMPILER} + export WLCLK=${WLCLK} + EOF + ;; + dbg) + CREATE_BASELINE=false + RTPWD=$NEW_BASELINE + BL_SUFFIX=_dbg_base + comp_nm=dbg + WLCLK=30 + cat <<-EOF >${RUNDIR_ROOT}/unit_test${RT_SUFFIX}.env + export UNIT_TEST=true + export NEMS_COMPILER=${NEMS_COMPILER} export WLCLK=${WLCLK} EOF ;; esac - BL_SUFFIX="_$comp_nm" - cat <<- EOF > ${RUNDIR_ROOT}/run_test${RT_SUFFIX}.env - export MACHINE_ID=${MACHINE_ID} - export RTPWD=${RTPWD} - export PATHRT=${PATHRT} - export PATHTR=${PATHTR} - export NEW_BASELINE=${NEW_BASELINE} - export CREATE_BASELINE=${CREATE_BASELINE} - export RT_SUFFIX=${RT_SUFFIX} - export BL_SUFFIX=${BL_SUFFIX} - export SCHEDULER=${SCHEDULER} - export ACCNR=${ACCNR} - export QUEUE=${QUEUE} - export ROCOTO=${ROCOTO} - export LOG_DIR=${LOG_DIR} - EOF + export MACHINE_ID=${MACHINE_ID} + export RTPWD=${RTPWD} + export PATHRT=${PATHRT} + export PATHTR=${PATHTR} + export NEW_BASELINE=${NEW_BASELINE} + export CREATE_BASELINE=${CREATE_BASELINE} + export RT_SUFFIX=${RT_SUFFIX} + export BL_SUFFIX=${BL_SUFFIX} + export SCHEDULER=${SCHEDULER} + export ACCNR=${ACCNR} + export QUEUE=${QUEUE} + export PARTITION=${PARTITION} + export ROCOTO=${ROCOTO} + export ECFLOW=${ECFLOW} + export REGRESSIONTEST_LOG=${REGRESSIONTEST_LOG} + export LOG_DIR=${LOG_DIR} + EOF if [[ $ECFLOW == true ]]; then TEST_NR=${RT_SUFFIX:1} COMPILE_NR=$comp_nm DEP_RUN= - if [[ ${dependency_flag} == all ]]; then - if [[ ${RT_SUFFIX} == _std || ${RT_SUFFIX} == _thread || ${RT_SUFFIX} == _mpi || ${RT_SUFFIX} == _decomp ]]; then - DEP_RUN="${TEST_NAME}_std_base" - elif [[ ${RT_SUFFIX} == _restart ]]; then - DEP_RUN="${TEST_NAME}_std" - elif [[ ${RT_SUFFIX} == _32bit ]]; then - DEP_RUN="${TEST_NAME}_32bit_base" - elif [[ ${RT_SUFFIX} == _debug ]]; then - DEP_RUN="${TEST_NAME}_debug_base" - fi - elif [[ ${dependency_flag} == restart ]]; then - if [[ ${RT_SUFFIX} == _restart ]]; then - DEP_RUN="${TEST_NAME}_std" - fi + if [[ ${RT_SUFFIX} == _std || ${RT_SUFFIX} == _thr || ${RT_SUFFIX} == _mpi || ${RT_SUFFIX} == _dcp || ${RT_SUFFIX} == _rst ]]; then + DEP_RUN="${TEST_NAME}_std_base" + elif [[ ${RT_SUFFIX} == _bit ]]; then + DEP_RUN="${TEST_NAME}_bit_base" + elif [[ ${RT_SUFFIX} == _dbg ]]; then + DEP_RUN="${TEST_NAME}_dbg_base" else DEP_RUN='' fi ecflow_create_run_task else echo "Running test for $rc with" - echo " THRD: $THRD; INPES: $INPES; JNPES: $JNPES; TASKS: $TASKS; TPN: $TPN; NODES: $NODES" + echo " THRD: $THRD; INPES: $INPES; JNPES: $JNPES; TASKS: $TASKS; TPN: $TPN" TEST_NR=${RT_SUFFIX:1} ./run_test.sh $PATHRT $RUNDIR_ROOT $TEST_NAME $TEST_NR $comp_nm > $LOG_DIR/run_${TEST_NAME}_${TEST_NR}.log 2>&1 fi @@ -248,11 +359,11 @@ run_utests() { done } -trap 'echo utest killed; cleanup' KILL -trap 'echo utest interrupted; cleanup' INT -trap 'echo utest quit; cleanup' QUIT -trap 'echo utest terminated; cleanup' TERM -trap 'echo utest error on line $LINENO; cleanup' ERR +trap 'echo utest killed; cleanup $?' KILL +trap 'echo utest interrupted; cleanup $?' INT +trap 'echo utest quit; cleanup $?' QUIT +trap 'echo utest terminated; cleanup $?' TERM +trap 'echo utest error on line $LINENO; cleanup $?' ERR trap ' ret=$? if [ "$ret" -ne 0 ]; then @@ -260,38 +371,44 @@ trap ' else echo >&2 "utest finished" fi - cleanup' EXIT + cleanup $ret' EXIT ######################################################################## #### PROGRAM STARTS #### ######################################################################## + readonly program=$(basename $0) +# PATHRT - Path to unit tests directory +readonly PATHRT=$(cd $(dirname $0) && pwd -P) +# PATHTR - Path to trunk directory +readonly PATHTR=$(cd ${PATHRT}/.. && pwd) +readonly lockdir=${PATHRT}/lock_ut export UNIT_TEST=true +export CI_TEST=${CI_TEST:-false} + [[ $# -eq 0 ]] && usage_and_exit 1 -# Default compiler: intel +# specify compiler +NEMS_COMPILER=${NEMS_COMPILER:-} export COMPILER=${NEMS_COMPILER:-intel} # detect_machine sets ACCNR and MACHINE_ID +NEMS_MACHINE=${NEMS_MACHINE:-} source detect_machine.sh -# utility functions in rt_utils need to be able to see variables in utest -source rt_utils.sh -# PATHRT - Path to unit tests directory -readonly PATHRT=$(cd $(dirname $0) && pwd -P) cd $PATHRT -# PATHTR - Path to trunk directory -readonly PATHTR=$(cd ${PATHRT}/.. && pwd) # make sure only one instance of utest is running -readonly lockdir=${PATHRT}/lock_ut if mkdir $lockdir 2>/dev/null; then echo $(hostname) $$ > ${lockdir}/PID else error "Only one instance of utest can be running at a time" fi +# utility functions in rt_utils need to be able to see variables in utest +source rt_utils.sh + # Machine-dependent libraries, modules, variables, etc. if [[ $MACHINE_ID = hera.* ]]; then - export NCEPLIBS=/scratch1/NCEPDEV/global/gwv/l819/lib + source $PATHTR/NEMS/src/conf/module-setup.sh.inc module use $PATHTR/modulefiles/${MACHINE_ID} module load fv3 @@ -303,70 +420,137 @@ if [[ $MACHINE_ID = hera.* ]]; then ECFLOW_START=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ecflow/bin/ecflow_start.sh ECF_PORT=$(( $(id -u) + 1500 )) - QUEUE=debug + PARTITION= + QUEUE=batch + COMPILE_QUEUE=batch dprefix=/scratch1/NCEPDEV DISKNM=$dprefix/nems/emc.nemspara/RT STMP=${dprefix}/stmp4 PTMP=${dprefix}/stmp2 SCHEDULER=slurm cp fv3_conf/fv3_slurm.IN_hera fv3_conf/fv3_slurm.IN + cp fv3_conf/compile_slurm.IN_hera fv3_conf/compile_slurm.IN + +elif [[ $MACHINE_ID = orion.* ]]; then + + source $PATHTR/NEMS/src/conf/module-setup.sh.inc + + module use $PATHTR/modulefiles/${MACHINE_ID} + module load fv3 + module load gcc/8.3.0 + + # Re-instantiate COMPILER in case it gets deleted by module purge + COMPILER=${NEMS_COMPILER:-intel} + + export PATH=/work/noaa/fv3-cam/djovic/ecflow/bin:$PATH + export PYTHONPATH=/work/noaa/fv3-cam/djovic/ecflow/lib/python2.7/site-packages + ECFLOW_START=/work/noaa/fv3-cam/djovic/ecflow/bin/ecflow_start.sh + ECF_PORT=$(( $(id -u) + 1500 )) + QUEUE=batch + COMPILE_QUEUE=batch +# ACCNR= # detected in detect_machine.sh + PARTITION=orion + dprefix=/work/noaa/stmp/${USER} + DISKNM=/work/noaa/nems/emc.nemspara/RT + STMP=$dprefix/stmp + PTMP=$dprefix/stmp + + SCHEDULER=slurm + cp fv3_conf/fv3_slurm.IN_orion fv3_conf/fv3_slurm.IN + cp fv3_conf/compile_slurm.IN_orion fv3_conf/compile_slurm.IN + +elif [[ $MACHINE_ID = wcoss_dell_p3 ]]; then + + source $PATHTR/NEMS/src/conf/module-setup.sh.inc + module load lsf/10.1 + + module use $PATHTR/modulefiles/${MACHINE_ID} + module load fv3 + + module load python/2.7.14 + + module load ips/18.0.1.163 + module load ecflow/4.7.1 + ECFLOW_START=${ECF_ROOT}/intel/bin/ecflow_start.sh + ECF_PORT=$(grep $USER /usrx/local/sys/ecflow/assigned_ports.txt | awk '{print $2}') + + DISKNM=/gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT + QUEUE=debug + COMPILE_QUEUE=dev_transfer + PARTITION= + ACCNR=GFS-DEV + STMP=/gpfs/dell2/stmp + PTMP=/gpfs/dell2/ptmp + SCHEDULER=lsf + cp fv3_conf/fv3_bsub.IN_wcoss_dell_p3 fv3_conf/fv3_bsub.IN + cp fv3_conf/compile_bsub.IN_wcoss_dell_p3 fv3_conf/compile_bsub.IN + +elif [[ $MACHINE_ID = linux.* ]]; then + + COMPILER=${NEMS_COMPILER:-intel} + PARTITION= + QUEUE= + COMPILE_QUEUE= + dprefix=/home/tester + DISKNM=${dprefix}/data + STMP=${dprefix}/stmp4 + PTMP=${dprefix}/stmp2 + SCHEDULER=none + else - error "Unknown machine ID. Edit detect_machine.sh file" + error "unknown machine ID. edit detect_machine.sh file" fi -CREATE_BASELINE=true -baseline_cases= -run_unit_test=true -unit_test_cases= +# default variable values TEST_NAME= -keep_rundir=false -ROCOTO=false +CREATE_BASELINE= +test_case="std thr mpi dcp rst bit dbg" ECFLOW=false +ROCOTO=false +std_compare=false +bit_compare=false +dbg_compare=false +keep_rundir=false +skip_compile=false +skip_run=false -# Parse command line arguments -while getopts :c:r:n:ekh opt; do +# parse command line arguments to fill-in/modify the above default variables +while getopts :n:c:ekhbdsxz opt; do case $opt in - c) - run_unit_test=false - if [ $OPTARG = all ]; then - baseline_cases=std,32bit,debug - else - baseline_cases=$OPTARG - fi - baseline_cases=$(echo $baseline_cases | sed -e 's/^ *//' -e 's/ *$//' -e 's/,/ /g') - for i in $baseline_cases - do - if [[ $i != std && $i != 32bit && $i != debug ]]; then - error "Invalid baseline_cases specified: $i" - fi - done + n) + TEST_NAME=$OPTARG + echo "test name: ${TEST_NAME}" ;; - r) - CREATE_BASELINE=false - if [ $OPTARG = all ]; then - unit_test_cases=std,thread,mpi,decomp,restart,32bit,debug - else - unit_test_cases=$OPTARG - fi - unit_test_cases=$(echo $unit_test_cases | sed -e 's/^ *//' -e 's/ *$//' -e 's/,/ /g') - for i in $unit_test_cases - do - if [[ $i != std && $i != thread && $i != mpi && $i != decomp && \ - $i != restart && $i != 32bit && $i != debug ]]; then - error "Invalid unit_test_cases specified: $i" + c) + test_case=$OPTARG + test_case=$(echo $test_case | sed -e 's/^ *//' -e 's/ *$//' -e 's/,/ /g') + for i in $test_case; do + if [[ $i != std && $i != thr && $i != mpi && $i != dcp && $i != rst && $i != bit && $i != dbg ]]; then + error "invalid test case specified: $i" fi done ;; - n) - TEST_NAME=$OPTARG - echo "Test name: ${TEST_NAME}" - ;; e) ECFLOW=true ;; k) keep_rundir=true ;; + b) + bit_compare=true + ;; + d) + dbg_compare=true + ;; + s) + std_compare=true + ;; + x) + skip_compile=true + ;; + z) + skip_run=true + ;; h) usage_and_exit 0 ;; @@ -376,94 +560,73 @@ while getopts :c:r:n:ekh opt; do esac done -# TEST_NAME is a required argument -if [ -z $TEST_NAME ]; then - error "$program: please specify test-name. Try 'utest -h' for usage." +# some safety guards for input arguments +if [[ -z $TEST_NAME ]]; then + error "$program: test name is required. try 'utest -h' for usage" fi - -# Default where neither -c nor -r is specified: compile and run all cases -if [[ $CREATE_BASELINE == true && $run_unit_test == true ]]; then - baseline_cases=std,32bit,debug - unit_test_cases=std,thread,mpi,decomp,restart,32bit,debug +if [[ $skip_compile == true && $skip_run == true ]]; then + error "$program: cannot skip both compile and run" fi - -#echo "baseline_cases = $baseline_cases" -#echo "unit_test_cases = $unit_test_cases" - -# Fill in ut_compile_cases & ut_run_cases based on baseline_cases & unit_test_cases -# Cases are sorted in the order: std,thread,mpi,decomp,restart,32bit,debug -ut_compile_cases= -ut_run_cases= -if [[ $CREATE_BASELINE == true && $run_unit_test == true ]]; then - ut_compile_cases="1std 232bit 3debug" - ut_run_cases="1std 2thread 3mpi 4decomp 5restart 632bit 7debug" -elif [[ $CREATE_BASELINE == true && $run_unit_test == false ]]; then - for i in $baseline_cases; do - case $i in - std) - ut_compile_cases+=" 1$i" - ut_run_cases+=" 1$i" - ;; - 32bit) - ut_compile_cases+=" 2$i" - ut_run_cases+=" 2$i" - ;; - debug) - ut_compile_cases+=" 3$i" - ut_run_cases+=" 3$i" - ;; - esac - done -elif [[ $run_unit_test == true && $CREATE_BASELINE == false ]]; then - for i in $unit_test_cases; do - case $i in - std) - ut_compile_cases+=" 1$i" - ut_run_cases+=" 1$i" - ;; - thread) - ut_compile_cases+=" 1std" - ut_run_cases+=" 2$i" - ;; - mpi) - ut_compile_cases+=" 1std" - ut_run_cases+=" 3$i" - ;; - decomp) - ut_compile_cases+=" 1std" - ut_run_cases+=" 4$i" - ;; - restart) - ut_compile_cases+=" 1std" - ut_run_cases+=" 5$i" - ;; - 32bit) - ut_compile_cases+=" 2$i" - ut_run_cases+=" 6$i" - ;; - debug) - ut_compile_cases+=" 3$i" - ut_run_cases+=" 7$i" - ;; - esac - done +if [[ $std_compare == true && ! $test_case =~ std ]]; then + error "$program: std reproducibility test requires specifying std" +fi +if [[ $bit_compare == true && ! $test_case =~ bit ]]; then + error "$program: bit reproducibility test requires specifying bit" fi -ut_compile_cases=$(echo $ut_compile_cases | tr " " "\n" | sort -u) -ut_compile_cases=$(echo $ut_compile_cases | sed -e 's/^[0-9]//g' -e 's/ [0-9]/ /g') -ut_run_cases=$(echo $ut_run_cases | tr " " "\n" | sort -u) -ut_run_cases=$(echo $ut_run_cases | sed -e 's/^[0-9]//g' -e 's/ [0-9]/ /g') -if [[ ! $ut_run_cases =~ ^std && $ut_run_cases =~ restart ]]; then - ut_run_cases="std ${ut_run_cases}" +if [[ $dbg_compare == true && ! $test_case =~ dbg ]]; then + error "$program: debug reproducibility test requires specifying dbg" fi -echo "Cases to compile: $ut_compile_cases" -echo "Cases to run : $ut_run_cases" -# Log directory +# enumerate which case to compile and run +compile_case= +run_case= +for i in $test_case; do + if [[ $i == thr || $i == mpi || $i == dcp || $i == rst ]]; then + if [[ ! $compile_case =~ std ]]; then + compile_case+=" std" + fi + if [[ ! $run_case =~ std_base ]]; then + run_case+=" std_base" + fi + run_case+=" $i" + elif [[ $i == std ]]; then + if [[ ! $compile_case =~ std ]]; then + compile_case+=" std" + fi + if [[ ! $run_case =~ std_base ]]; then + run_case+=" std_base" + fi + if [[ $std_compare == true ]]; then + run_case+=" std" + fi + elif [[ $i == bit ]]; then + compile_case+=" bit" + run_case+=" bit_base" + if [[ $bit_compare == true ]]; then + run_case+=" bit" + fi + elif [[ $i == dbg ]]; then + compile_case+=" dbg" + run_case+=" dbg_base" + if [[ $dbg_compare == true ]]; then + run_case+=" dbg" + fi + fi +done +compile_case=$(echo $compile_case | sed -e 's/^ *//' -e 's/ *$//') +run_case=$(echo $run_case | sed -e 's/^ *//' -e 's/ *$//') +# if there exists std_base in run_case, make sure it is the first +if [[ $run_case =~ std_base && ! $run_case =~ ^std_base ]]; then + run_case=$(echo $run_case | sed -e 's/ std_base//') + run_case="std_base $run_case" +fi + +# log directory LOG_DIR=${PATHRT}/log_ut_$MACHINE_ID rm -rf ${LOG_DIR} mkdir ${LOG_DIR} -# Directory where all simulations are run +# directory where all simulations are run RUNDIR_ROOT=${RUNDIR_ROOT:-${PTMP}/${USER}}/FV3_UT/ut_$$ mkdir -p ${RUNDIR_ROOT} @@ -474,127 +637,103 @@ if [[ $ECFLOW == true ]]; then mkdir -p ${ECFLOW_RUN}/${ECFLOW_SUITE} cp head.h tail.h ${ECFLOW_RUN} > ${ECFLOW_RUN}/${ECFLOW_SUITE}.def - cat << EOF >> ${ECFLOW_RUN}/${ECFLOW_SUITE}.def -suite ${ECFLOW_SUITE} - edit ECF_HOME '${ECFLOW_RUN}' - edit ECF_INCLUDE '${ECFLOW_RUN}' - edit ECF_KILL_CMD kill -15 %ECF_RID% > %ECF_JOB%.kill 2>&1 - edit ECF_TRIES 1 - label rundir_root '${RUNDIR_ROOT}' - limit max_builds 6 - limit max_jobs 30 -EOF + cat <<-EOF >> ${ECFLOW_RUN}/${ECFLOW_SUITE}.def + suite ${ECFLOW_SUITE} + edit ECF_HOME '${ECFLOW_RUN}' + edit ECF_INCLUDE '${ECFLOW_RUN}' + edit ECF_KILL_CMD kill -15 %ECF_RID% > %ECF_JOB%.kill 2>&1 + edit ECF_TRIES 1 + label rundir_root '${RUNDIR_ROOT}' + limit max_builds 6 + limit max_jobs 30 + EOF if [[ $MACHINE_ID == hera.* ]]; then - QUEUE=debug + QUEUE=batch + elif [[ $MACHINE_ID == orion.* ]]; then + QUEUE=batch + elif [[ $MACHINE_ID == wcoss_dell_p3 ]]; then + QUEUE=dev else error "ecflow is not supported on this machine $MACHINE_ID" fi fi +compile_log=${PATHRT}/Compile_ut_$MACHINE_ID.log +unittest_log=${PATHRT}/UnitTests_$MACHINE_ID.log +REGRESSIONTEST_LOG=$unittest_log + ######################################################################## #### COMPILE #### ######################################################################## + # build_file specifies compilation options build_file='utest.bld' -[[ -f ${build_file} ]] || error "${build_file} does not exist" -compile_log=${PATHRT}/Compile_ut_$MACHINE_ID.log -rm -f fv3_*.exe modules.fv3_* ${compile_log} +# if skipping compile, check if all required executables exist. if not, +# exit with a non-zero number equal to the number of missing executables +num_of_missing_exe=0 +if [[ $skip_compile == true ]]; then + for i in $compile_case; do + if [[ ! -f ${PATHRT}/fv3_$i.exe ]]; then + echo "cannot find the required executable file fv3_$i.exe" >&2 + num_of_missing_exe=$(( num_of_missing_exe + 1 )) + else + echo "found the required executable file fv3_$i.exe" >&2 + fi + done -while IFS="|" read model comp_opt; do - model_found=false - model=$(echo $model | sed -e 's/^ *//' -e 's/ *$//') - comp_opt=$(echo $comp_opt | sed -e 's/^ *//' -e 's/ *$//') - if [[ $model == ${TEST_NAME} ]]; then - base_opt=${comp_opt} - model_found=true - break + if [[ $num_of_missing_exe == 0 ]]; then + echo "all required executables are found" >&2 + else + exit $num_of_missing_exe + fi +else + echo "cases to compile: $compile_case" + build_utests + if [[ $ECFLOW == false ]]; then + cat ${LOG_DIR}/compile_*.log > ${compile_log} fi -done < ${build_file} -if [[ ${model_found} == false ]]; then - error "Build options for $TEST_NAME not found. Please edit utest.bld." fi -for name in $ut_compile_cases; do - case $name in - std) - NEMS_VER=${base_opt} - ;; - 32bit) - if [[ ${base_opt} =~ "32BIT=Y" ]]; then - NEMS_VER=$(echo ${base_opt} | sed -e 's/32BIT=Y/32BIT=N/') - elif [[ ${base_opt} =~ "32BIT=N" ]]; then - NEMS_VER=$(echo ${base_opt} | sed -e 's/32BIT=N/32BIT=Y/') - else - NEMS_VER="${base_opt} 32BIT=Y" - fi - ;; - debug) - NEMS_VER="${base_opt} 32BIT=Y DEBUG=Y" - ;; - esac - NEMS_VER=$(echo ${NEMS_VER} | sed -e 's/^ *//' -e 's/ *$//') +# if skipping run, stop after build. print out status information and +# exit with a non-zero number equal to the number of failed builds +num_of_failed_compilation=0 +if [[ $skip_run == true ]]; then + for i in $compile_case; do + if [[ ! -f ${PATHRT}/fv3_$i.exe ]]; then + echo "failed to generate executable file fv3_$i.exe" >&2 + num_of_failed_compilation=$(( num_of_failed_compilation + 1 )) + fi + done - if [[ $ECFLOW == true ]]; then - COMPILE_NR=$name - APP='' - ecflow_create_compile_task - else - echo "Compiling for ${name} with compile option ${NEMS_VER}" - ./compile.sh $PATHTR/FV3 $MACHINE_ID "${NEMS_VER}" $name >${LOG_DIR}/compile_${TEST_NAME}_$name.log 2>&1 - echo "Compiling done for ${name}" + if [[ $num_of_failed_compilation == 0 ]]; then + echo "all executables have been generated" >&2 fi -done + + exit $num_of_failed_compilation +fi ######################################################################## #### RUN #### ######################################################################## + mkdir -p ${STMP}/${USER} NEW_BASELINE=${STMP}/${USER}/FV3_UT/UNIT_TEST -RTPWD=$DISKNM/NEMSfv3gfs/develop-20200512/INTEL -if [[ $CREATE_BASELINE == true ]]; then - rm -rf $NEW_BASELINE - mkdir -p $NEW_BASELINE - - rsync -a "${RTPWD}"/FV3_* "${NEW_BASELINE}"/ - rsync -a "${RTPWD}"/WW3_* "${NEW_BASELINE}"/ +if [[ $MACHINE_ID == hera.* || $MACHINE_ID == orion.* ]]; then + baseline_location=$DISKNM/NEMSfv3gfs/develop-20200713/${COMPILER^^} +else + baseline_location=$DISKNM/NEMSfv3gfs/develop-20200713 fi +RTPWD=$baseline_location + +rm -rf $NEW_BASELINE +mkdir -p $NEW_BASELINE +ln -s ${RTPWD}/FV3_input_data ${NEW_BASELINE} -# unittest_log is different from REGRESSIONTEST_LOG -# defined in run_test.sh and passed onto rt_utils.sh -unittest_log=${PATHRT}/UnitTests_$MACHINE_ID.log rm -f fail_unit_test ${unittest_log} -create_flag= -dependency_flag= - -if [[ $CREATE_BASELINE == true && $run_unit_test == true ]]; then - # Run to create baseline - create_flag=true - dependency_flag=none - ut_run_cases='std 32bit debug' - run_utests - # Run to compare with baseline - create_flag=false - dependency_flag=all - ut_run_cases='std thread mpi decomp restart 32bit debug' - CREATE_BASELINE=false - RTPWD=${NEW_BASELINE} - run_utests -elif [[ $CREATE_BASELINE == true && $run_unit_test == false ]]; then - # Run to create baseline - create_flag=true - dependency_flag=none - run_utests -elif [[ $CREATE_BASELINE == false && $run_unit_test == true ]]; then - # Run to compare with baseline - create_flag=false - dependency_flag=restart - if [[ ! -d $NEW_BASELINE ]]; then - error "There is no baseline to run unit tests against. Create baselines first." - fi - RTPWD=${NEW_BASELINE} - run_utests -fi + +echo "cases to run: $run_case" +run_utests if [[ $ECFLOW == true ]]; then echo "endsuite" >> ${ECFLOW_RUN}/${ECFLOW_SUITE}.def @@ -605,23 +744,32 @@ fi #### UNIT TEST STATUS #### ######################################################################## set +e -cat ${LOG_DIR}/compile_*.log > ${compile_log} -cat ${LOG_DIR}/ut_*.log >> ${unittest_log} +if [[ $ECFLOW == true ]]; then + cat ${LOG_DIR}/compile_*.log > ${compile_log} +fi +cat ${LOG_DIR}/ut_*.log | tee -a ${unittest_log} if [[ -e fail_unit_test ]]; then echo "FAILED TESTS: " | tee -a ${unittest_log} - while read -r failed_test_name - do + while read -r failed_test_name; do echo "Test ${failed_test_name} failed " | tee -a ${unittest_log} done < fail_unit_test + echo "UNIT TEST FAILED" | tee -a ${unittest_log} + date >>${unittest_log} + hrs=$(( $SECONDS%86400/3600 )); mins=$(( $SECONDS%3600/60 )); secs=$(( $SECONDS%60 )) + elapsed_time=$(printf '%02dh:%02dm:%02ds\n' $hrs $mins $secs) + echo "Elapsed time: ${elapsed_time}. Have a nice day!" | tee -a ${unittest_log} + + exit 1 else + [[ ${keep_rundir} == false ]] && rm -rf ${RUNDIR_ROOT} + echo "UNIT TEST WAS SUCCESSFUL" | tee -a ${unittest_log} + date >>${unittest_log} + hrs=$(( $SECONDS%86400/3600 )); mins=$(( $SECONDS%3600/60 )); secs=$(( $SECONDS%60 )) + elapsed_time=$(printf '%02dh:%02dm:%02ds\n' $hrs $mins $secs) + echo "Elapsed time: ${elapsed_time}. Have a nice day!" | tee -a ${unittest_log} - rm -f fv3_*.x fv3_*.exe modules.fv3_* - [[ ${keep_rundir} == false ]] && rm -rf ${RUNDIR_ROOT} + exit 0 fi - -date >> ${unittest_log} -elapsed_time=$(printf '%02dh:%02dm:%02ds\n' $(($SECONDS%86400/3600)) $(($SECONDS%3600/60)) $(($SECONDS%60))) -echo "Elapsed time: ${elapsed_time}. Have a nice day!" | tee -a ${unittest_log} diff --git a/tests/utest.bld b/tests/utest.bld index 2fca9fc014..3d92bc71c2 100644 --- a/tests/utest.bld +++ b/tests/utest.bld @@ -1,28 +1,23 @@ -fv3_control | -fv3_gfdlmprad_gwd | -fv3_thompson | -fv3_gfdlmp | -fv3_gfdlmprad_noahmp | -fv3_wsm6 | -fv3_stochy | -fv3_wrtGauss_netcdf_esmf | -fv3_wrtGauss_netcdf | -fv3_wrtGauss_nemsio | -fv3_csawmg | -fv3_csawmgshoc | -fv3_satmedmf | -fv3_lheatstrg | -fv3_iau | -fv3_csawmg3shoc127 | -fv3_gfdlmprad | WW3=Y fv3_ccpp_control | CCPP=Y SUITES=FV3_GFS_2017 fv3_ccpp_wrtGauss_netcdf_esmf | CCPP=Y SUITES=FV3_GFS_2017 fv3_ccpp_wrtGauss_netcdf | CCPP=Y SUITES=FV3_GFS_2017 +fv3_ccpp_wrtGlatlon_netcdf | CCPP=Y SUITES=FV3_GFS_2017 fv3_ccpp_wrtGauss_nemsio | CCPP=Y SUITES=FV3_GFS_2017 -fv3_ccpp_wrtGauss_nemsio_c192 | CCPP=Y SUITES=FV3_GFS_2017 fv3_ccpp_stochy | CCPP=Y SUITES=FV3_GFS_2017 +fv3_ccpp_iau | CCPP=Y SUITES=FV3_GFS_2017 +fv3_ccpp_ca | CCPP=Y SUITES=FV3_GFS_2017 +fv3_ccpp_lheatstrg | CCPP=Y SUITES=FV3_GFS_2017 +fv3_ccpp_gfdlmprad | CCPP=Y SUITES=FV3_GFS_2017,FV3_GFS_2017_gfdlmp WW3=Y +fv3_ccpp_atmwav | CCPP=Y SUITES=FV3_GFS_2017,FV3_GFS_2017_gfdlmp WW3=Y +fv3_ccpp_multigases | CCPP=Y SUITES=FV3_GFS_2017_fv3wam 32BIT=Y MULTI_GASES=Y fv3_ccpp_gfdlmp | CCPP=Y SUITES=FV3_GFS_2017_gfdlmp fv3_ccpp_gfdlmprad_gwd | CCPP=Y SUITES=FV3_GFS_2017_gfdlmp fv3_ccpp_gfdlmprad_noahmp | CCPP=Y SUITES=FV3_GFS_2017_gfdlmp_noahmp +fv3_ccpp_gfdlmp_hwrfsas | CCPP=Y SUITES=FV3_GFS_2017_gfdlmp,FV3_GFS_2017_gfdlmp_noahmp fv3_ccpp_csawmg | CCPP=Y SUITES=FV3_GFS_2017_csawmg fv3_ccpp_satmedmf | CCPP=Y SUITES=FV3_GFS_2017_satmedmf +fv3_ccpp_satmedmfq | CCPP=Y SUITES=FV3_GFS_2017_satmedmfq +fv3_ccpp_gfsv16_csawmg | CCPP=Y SUITES=FV3_GFS_v16_csawmg +fv3_ccpp_gfsv16_csawmgt | CCPP=Y SUITES=FV3_GFS_v16_csawmg +fv3_ccpp_gocart_clm | CCPP=Y SUITES=FV3_GFS_2017_gfdlmp,FV3_GFS_2017_gfdlmp_noahmp,FV3_GFS_v16beta_flake +fv3_ccpp_gfs_v16beta_flake | CCPP=Y SUITES=FV3_GFS_2017_gfdlmp,FV3_GFS_2017_gfdlmp_noahmp,FV3_GFS_v16beta_flake