From 0d8a66f591089a3fad068f4bded25616464eab99 Mon Sep 17 00:00:00 2001 From: Daniel Rosen Date: Tue, 28 May 2024 13:49:32 -0600 Subject: [PATCH] Add ctest for ESMX washita_dlnd --- CMakeLists.txt | 9 +- cmake/modules/ParflowTest.cmake | 17 +++ pfnuopc/CMakeLists.txt | 18 +++ pfnuopc/esmxBuild.yaml | 9 ++ test/esmx/CMakeLists.txt | 3 + test/esmx/INSTRUCTIONS.md | 111 ------------------ test/esmx/README.md | 12 ++ test/esmx/pbs_template.sh | 45 ------- test/esmx/slurm_template.sh | 46 -------- .../{Little_Washita.py => washita_dlnd.py} | 18 +-- test/esmx/{esmxRun.yaml => washita_dlnd.yaml} | 4 +- .../PARFLOW_INPUTS => input/esmx}/LW.mask.pfb | Bin .../PARFLOW_INPUTS => input/esmx}/LW.pfsol | 0 .../esmx/LW_Indicator_USGS_Bedrock.pfb} | Bin .../esmx/LW_drv_clmin.dat} | 0 .../esmx/LW_drv_vegm.alluv.dat} | 0 .../esmx/LW_drv_vegp.dat} | 0 .../esmx/LW_press.init.233.pfb} | Bin .../esmx/LW_slopex.pfb} | Bin .../esmx/LW_slopey.pfb} | Bin 20 files changed, 77 insertions(+), 215 deletions(-) create mode 100644 pfnuopc/esmxBuild.yaml create mode 100644 test/esmx/CMakeLists.txt delete mode 100644 test/esmx/INSTRUCTIONS.md create mode 100644 test/esmx/README.md delete mode 100644 test/esmx/pbs_template.sh delete mode 100644 test/esmx/slurm_template.sh rename test/esmx/{Little_Washita.py => washita_dlnd.py} (98%) rename test/esmx/{esmxRun.yaml => washita_dlnd.yaml} (96%) rename test/{esmx/PARFLOW_INPUTS => input/esmx}/LW.mask.pfb (100%) rename test/{esmx/PARFLOW_INPUTS => input/esmx}/LW.pfsol (100%) rename test/{esmx/PARFLOW_INPUTS/Indicator_LW_USGS_Bedrock.pfb => input/esmx/LW_Indicator_USGS_Bedrock.pfb} (100%) rename test/{esmx/PARFLOW_INPUTS/drv_clmin.dat => input/esmx/LW_drv_clmin.dat} (100%) rename test/{esmx/PARFLOW_INPUTS/drv_vegm.alluv.dat => input/esmx/LW_drv_vegm.alluv.dat} (100%) rename test/{esmx/PARFLOW_INPUTS/drv_vegp.dat => input/esmx/LW_drv_vegp.dat} (100%) rename test/{esmx/PARFLOW_INPUTS/press.init.233.pfb => input/esmx/LW_press.init.233.pfb} (100%) rename test/{esmx/PARFLOW_INPUTS/slopex_LW.pfb => input/esmx/LW_slopex.pfb} (100%) rename test/{esmx/PARFLOW_INPUTS/slopey_LW.pfb => input/esmx/LW_slopey.pfb} (100%) diff --git a/CMakeLists.txt b/CMakeLists.txt index a86511390..3d77ad990 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -613,10 +613,15 @@ if ( ${PARFLOW_ENABLE_TOOLS} ) add_subdirectory (pftools) endif () -# Optionally build the NUOPC cap +# Optionally build the NUOPC cap and ESMX option(PARFLOW_ENABLE_NUOPC "Enable building of the Parflow nuopc" "False") -if ( ${PARFLOW_ENABLE_NUOPC} ) +option(PARFLOW_ENABLE_ESMX "Enable building of ESMX" "False") +if ( ${PARFLOW_ENABLE_NUOPC} OR ${PARFLOW_ENABLE_ESMX} ) add_subdirectory (pfnuopc) + if ( ${PARFLOW_ENABLE_ESMX} ) + set(PARFLOW_ESMX "${CMAKE_BINARY_DIR}/esmx/esmx_parflow") + add_subdirectory(test/esmx) + endif () endif () #----------------------------------------------------------------------------- diff --git a/cmake/modules/ParflowTest.cmake b/cmake/modules/ParflowTest.cmake index d08b5ced1..a3f7411d9 100644 --- a/cmake/modules/ParflowTest.cmake +++ b/cmake/modules/ParflowTest.cmake @@ -130,3 +130,20 @@ function (pf_add_py_parallel_test test_name topology_P topology_Q topology_R) ) endfunction() + +# Add parflow testing of a single ESMX file +function (pf_add_esmx_test test_name) + + configure_file(fd.yaml ${CMAKE_CURRENT_BINARY_DIR} COPYONLY) + configure_file(${test_name}.py ${CMAKE_CURRENT_BINARY_DIR} COPYONLY) + add_test( + NAME "esmx_${test_name}" + COMMAND ${PARFLOW_ESMX} "${CMAKE_CURRENT_SOURCE_DIR}/${test_name}.yaml" + ) + set_tests_properties( + "esmx_${test_name}" + PROPERTIES + ENVIRONMENT "PF_SRC=${PROJECT_SOURCE_DIR}" + ) + +endfunction() diff --git a/pfnuopc/CMakeLists.txt b/pfnuopc/CMakeLists.txt index 36fb2b4a4..5388f7a1d 100644 --- a/pfnuopc/CMakeLists.txt +++ b/pfnuopc/CMakeLists.txt @@ -33,6 +33,24 @@ include_directories(parflow_nuopc include) target_link_libraries(parflow_nuopc PUBLIC ESMF) target_link_libraries(parflow_nuopc PUBLIC pfsimulator) +# configure fire behavior esmx executable +if (PARFLOW_ENABLE_ESMX) + if(ESMF_VERSION VERSION_LESS "8.5.0") + message(FATAL_ERROR "ESMX requires ESMF version 8.5.0 or greater") + endif (ESMF_VERSION VERSION_LESS "8.5.0") + include(ExternalProject) + ExternalProject_Add(esmx + BINARY_DIR ${CMAKE_BINARY_DIR}/esmx + SOURCE_DIR ${ESMF_ESMXDIR} + CMAKE_ARGS -DESMX_BUILD_FILE=${CMAKE_CURRENT_SOURCE_DIR}/esmxBuild.yaml + -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX} + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + BUILD_ALWAYS TRUE + DEPENDS parflow_nuopc + EXCLUDE_FROM_ALL TRUE + ) +endif (PARFLOW_ENABLE_ESMX) + ### Install Library install(TARGETS parflow_nuopc EXPORT ParFlow diff --git a/pfnuopc/esmxBuild.yaml b/pfnuopc/esmxBuild.yaml new file mode 100644 index 000000000..445c83563 --- /dev/null +++ b/pfnuopc/esmxBuild.yaml @@ -0,0 +1,9 @@ +application: + exe_name: esmx_parflow + +components: + parflow: + build_type: none + cmake_config: ParFlow.cmake + libraries: parflow_nuopc + fort_module: parflow_nuopc.mod diff --git a/test/esmx/CMakeLists.txt b/test/esmx/CMakeLists.txt new file mode 100644 index 000000000..818ef52c5 --- /dev/null +++ b/test/esmx/CMakeLists.txt @@ -0,0 +1,3 @@ +include(ParflowTest) + +pf_add_esmx_test(washita_dlnd) diff --git a/test/esmx/INSTRUCTIONS.md b/test/esmx/INSTRUCTIONS.md deleted file mode 100644 index 0ff57322b..000000000 --- a/test/esmx/INSTRUCTIONS.md +++ /dev/null @@ -1,111 +0,0 @@ -# ESMX ParFlow Test Application Instructions - -## Build -Include ParFlow in ESMX Build configuration file -``` - ParFlow: - fort_module: parflow_nuopc.mod - libraries: parflow_nuopc - build_args: -DPARFLOW_ENABLE_NUOPC=ON -DPARFLOW_AMPS_LAYER=mpi1 -DPARFLOW_HAVE_CLM=ON -DPARFLOW_ENABLE_HYPRE=ON -DPARFLOW_ENABLE_SILO=ON -DPARFLOW_ENABLE_NETCDF=ON - test_dir: ParFlow/test/esmx -``` - -## Configuration -Edit esmxRun.config - -### Component List -Provide list of component names to ESMX Driver. -``` -ESMX_component_list: LND GWR -``` - -### ESMX Data Component -Configure the ESMX Data component -``` -LND_model: ESMX_Data -``` - -| LND\_attributes | Value | Description | -| --------------- | ------------------------ | ----------------------------- | -| Verbosity | max | Enable all generic output | -| Diagnostic | 0 | No diagnostic output | -| coordSys | ESMF\_COORDSYS\_SPH\_DEG | Set coordinates using degrees | -| minx | -98.426653 | Western longitude of LW | -| maxx | -97.718663 | Eastern longitude of LW | -| miny | 34.739932 | Southern latitude of LW | -| maxy | 35.031552 | Northern latitude of LW | - -| LND\_import\_fields | dim | min | max | -| ---------------------------------------------- | --- | --- | ------ | -| soil\_moisture\_fraction\_layer\_1 | 2 | 0 | 1 | -| soil\_moisture\_fraction\_layer\_2 | 2 | 0 | 1 | -| soil\_moisture\_fraction\_layer\_3 | 2 | 0 | 1 | -| soil\_moisture\_fraction\_layer\_4 | 2 | 0 | 1 | -| liquid\_fraction\_of\_soil\_moisture\_layer\_1 | 2 | 0 | 1 | -| liquid\_fraction\_of\_soil\_moisture\_layer\_2 | 2 | 0 | 1 | -| liquid\_fraction\_of\_soil\_moisture\_layer\_3 | 2 | 0 | 1 | -| liquid\_fraction\_of\_soil\_moisture\_layer\_4 | 2 | 0 | 1 | -| ground\_water\_storage | 2 | 0 | 999999 | - -| LND\_export\_fields | dim | fill value | -| ---------------------------- | --- | ------------------ | -| precip\_drip | 2 | 0.000024 | -| bare\_soil\_evaporation | 2 | 0.000012 | -| total\_water\_flux\_layer\_1 | 2 | -0.000024000009596 | -| total\_water\_flux\_layer\_2 | 2 | -0.000000000004798 | -| total\_water\_flux\_layer\_3 | 2 | -0.000000000004798 | -| total\_water\_flux\_layer\_4 | 2 | -0.000000000004798 | - -### ParFlow Component -Configure the ParFlow component -``` -GWR_model: ParFlow -``` - -| HYD\_attributes | Description | -| -------------------- | --------------------------------------------- | -| Verbosity | integer interpreted as a bit field | -| Diagnostic | integer interpreted as a bit field | -| realize\_all\_import | true, false | -| realize\_all\_export | true, false | -| prep\_filename | ParFlow preprocessor configuration file | -| filename | ParFlow configuration file | -| initialize\_import | true, false | -| initialize\_export | true, false | -| check\_import | check import time stamp | -| coord\_type | GRD\_COORD\_CARTESIAN or GRD\_COORD\_CLMVEGTF | -| coord\_filename | file used when coordinates loaded from file | -| output\_directory | ParFlow cap output directory | - - -## Execution -``` -mpirun -np 4 ./esmx -``` - -## Slurm Workload Manager -1. edit slurm\_template.sh - - partition=\ - - account=\ - - constraint=\ - - qos=\ - - setup environment as needed -``` -sbatch slurm\_template.sh -``` - -## PBS Workload Manager -1. edit pbs\_template.sh - - \-A \ - - \-q \ - - setup environment as needed -``` -qsub pbs\_template.sh -``` - -## Validation -Successful execution produces evaptrans, press, and satur files through 00003. - -[Slurm Documentation](https://slurm.schedmd.com/documentation.html) - -[PBS Pro Documentation](https://www.altair.com/pbs-works-documentation) diff --git a/test/esmx/README.md b/test/esmx/README.md new file mode 100644 index 000000000..5994e6723 --- /dev/null +++ b/test/esmx/README.md @@ -0,0 +1,12 @@ +# parflow/test/esmx + +This directory contains tools for testing ESMX applications. Each test test +pairs a yaml file used to configure ESMX at runtime and a matching python file +used to configure ParFlow. The field dictionary file (fd.yaml) is shared for +all ESMX tests. + +- washita_dlnd + +### shared_resources + +- fd.yaml diff --git a/test/esmx/pbs_template.sh b/test/esmx/pbs_template.sh deleted file mode 100644 index ea9b03dea..000000000 --- a/test/esmx/pbs_template.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/bash -#PBS -N lnd-hyd-test -#PBS -A -#PBS -l walltime=00:10:00 -#PBS -q -#PBS -j oe -#PBS -o lndhydtest.log -#PBS -l select=1:mpiprocs=1 - -# environment settings -# TODO - -# set umask -umask 022 -# set limits -ulimit -t unlimited -ulimit -f unlimited -ulimit -d unlimited -ulimit -s unlimited -ulimit -c unlimited - -echo "Job ID: ${PBS_JOBID}" - -s_tm=`date +%s` -s_hr=`date +%H`; s_mn=`date +%M`; s_sc=`date +%S` -echo "Model Start ${s_hr}:${s_mn}:${s_sc}" - -mpiexec_mpt -n 1 ./esmx -exec_s=$? - -e_tm=`date +%s` -e_hr=`date +%H`; e_mn=`date +%M`; e_sc=`date +%S` -echo "Model End ${e_hr}:${e_mn}:${e_sc}" - -r_tm=$(( ${e_tm} - ${s_tm} )) -r_hr=$(( ${r_tm} / 3600 )) -r_mn=$(( ${r_tm} % 3600 / 60 )) -r_sc=$(( ${r_tm} % 60 )) -echo "Model Runtime ${r_hr}:${r_mn}:${r_sc}" - -if [ $exec_s -ne 0 ]; then - echo "RESULT: ERROR ${exec_s}" -else - echo "RESULT: SUCCESS" -fi diff --git a/test/esmx/slurm_template.sh b/test/esmx/slurm_template.sh deleted file mode 100644 index 95896f584..000000000 --- a/test/esmx/slurm_template.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=lnd-hyd-test -#SBATCH --ntasks=1 -#SBATCH --time=00:10:00 -#SBATCH --partition= -#SBATCH --account= -#SBATCH --constraint= -#SBATCH --qos= -#SBATCH --output=lndhydtest.%j.log - -# environment settings -# TODO - -# set umask -umask 022 -# set limits -ulimit -t unlimited -ulimit -f unlimited -ulimit -d unlimited -ulimit -s unlimited -ulimit -c unlimited - -echo "Job ID: ${SLURM_JOB_ID}" - -s_tm=`date +%s` -s_hr=`date +%H`; s_mn=`date +%M`; s_sc=`date +%S` -echo "Model Start ${s_hr}:${s_mn}:${s_sc}" - -mpirun -np 1 ./esmx -exec_s=$? - -e_tm=`date +%s` -e_hr=`date +%H`; e_mn=`date +%M`; e_sc=`date +%S` -echo "Model End ${e_hr}:${e_mn}:${e_sc}" - -r_tm=$(( ${e_tm} - ${s_tm} )) -r_hr=$(( ${r_tm} / 3600 )) -r_mn=$(( ${r_tm} % 3600 / 60 )) -r_sc=$(( ${r_tm} % 60 )) -echo "Model Runtime ${r_hr}:${r_mn}:${r_sc}" - -if [ $exec_s -ne 0 ]; then - echo "RESULT: ERROR ${exec_s}" -else - echo "RESULT: SUCCESS" -fi diff --git a/test/esmx/Little_Washita.py b/test/esmx/washita_dlnd.py similarity index 98% rename from test/esmx/Little_Washita.py rename to test/esmx/washita_dlnd.py index 9e8d5cb0b..7a0dc01da 100644 --- a/test/esmx/Little_Washita.py +++ b/test/esmx/washita_dlnd.py @@ -24,16 +24,16 @@ script_path = get_absolute_path('.') + '/' ### current folder for py script -input_path = './PARFLOW_INPUTS/' -forcing_path = './PARFLOW_FORCING/NLDAS_LW/' +input_path = '$PF_SRC/test/input/esmx/' +#forcing_path = './' clm_output_path = './' pf_output_path = './' domain_file = 'LW.pfsol' -subsurface_file = 'Indicator_LW_USGS_Bedrock.pfb' -slope_x_file = 'slopex_LW.pfb' -slope_y_file = 'slopey_LW.pfb' -initial_file = 'press.init.233.pfb' +subsurface_file = 'LW_Indicator_USGS_Bedrock.pfb' +slope_x_file = 'LW_slopex.pfb' +slope_y_file = 'LW_slopey.pfb' +initial_file = 'LW_press.init.233.pfb' start_time = 0 stop_time = 192 @@ -64,9 +64,9 @@ cp( input_path + initial_file ) ### initial file -cp( input_path + 'drv_clmin.dat', 'drv_clmin.dat' ) -cp( input_path + 'drv_vegm.alluv.dat', 'drv_vegm.alluv.dat' ) -cp( input_path + 'drv_vegp.dat', 'drv_vegp.dat' ) +cp( input_path + 'LW_drv_clmin.dat', 'LW_drv_clmin.dat' ) +cp( input_path + 'LW_drv_vegm.alluv.dat', 'LW_drv_vegm.alluv.dat' ) +cp( input_path + 'LW_drv_vegp.dat', 'LW_drv_vegp.dat' ) #----------------------------------------------------------------------------------------- # Setup timing info diff --git a/test/esmx/esmxRun.yaml b/test/esmx/washita_dlnd.yaml similarity index 96% rename from test/esmx/esmxRun.yaml rename to test/esmx/washita_dlnd.yaml index e7ce9284a..8643b6d62 100644 --- a/test/esmx/esmxRun.yaml +++ b/test/esmx/washita_dlnd.yaml @@ -28,13 +28,13 @@ GWR: model: ParFlow petList: [0-0] attributes: - prep_filename: 'Little_Washita.py' + prep_filename: 'washita_dlnd.py' filename: 'LW.pfidb' initialize_import: 'FLD_INIT_FILLV' initialize_export: 'FLD_INIT_ZERO' check_import: 'FLD_CHECK_NEXTT' coord_type: 'GRD_COORD_CLMVEGTF' - coord_filename: 'drv_vegm.alluv.dat' + coord_filename: 'LW_drv_vegm.alluv.dat' output_directory: './GWR_CAP_OUTPUT' LND: diff --git a/test/esmx/PARFLOW_INPUTS/LW.mask.pfb b/test/input/esmx/LW.mask.pfb similarity index 100% rename from test/esmx/PARFLOW_INPUTS/LW.mask.pfb rename to test/input/esmx/LW.mask.pfb diff --git a/test/esmx/PARFLOW_INPUTS/LW.pfsol b/test/input/esmx/LW.pfsol similarity index 100% rename from test/esmx/PARFLOW_INPUTS/LW.pfsol rename to test/input/esmx/LW.pfsol diff --git a/test/esmx/PARFLOW_INPUTS/Indicator_LW_USGS_Bedrock.pfb b/test/input/esmx/LW_Indicator_USGS_Bedrock.pfb similarity index 100% rename from test/esmx/PARFLOW_INPUTS/Indicator_LW_USGS_Bedrock.pfb rename to test/input/esmx/LW_Indicator_USGS_Bedrock.pfb diff --git a/test/esmx/PARFLOW_INPUTS/drv_clmin.dat b/test/input/esmx/LW_drv_clmin.dat similarity index 100% rename from test/esmx/PARFLOW_INPUTS/drv_clmin.dat rename to test/input/esmx/LW_drv_clmin.dat diff --git a/test/esmx/PARFLOW_INPUTS/drv_vegm.alluv.dat b/test/input/esmx/LW_drv_vegm.alluv.dat similarity index 100% rename from test/esmx/PARFLOW_INPUTS/drv_vegm.alluv.dat rename to test/input/esmx/LW_drv_vegm.alluv.dat diff --git a/test/esmx/PARFLOW_INPUTS/drv_vegp.dat b/test/input/esmx/LW_drv_vegp.dat similarity index 100% rename from test/esmx/PARFLOW_INPUTS/drv_vegp.dat rename to test/input/esmx/LW_drv_vegp.dat diff --git a/test/esmx/PARFLOW_INPUTS/press.init.233.pfb b/test/input/esmx/LW_press.init.233.pfb similarity index 100% rename from test/esmx/PARFLOW_INPUTS/press.init.233.pfb rename to test/input/esmx/LW_press.init.233.pfb diff --git a/test/esmx/PARFLOW_INPUTS/slopex_LW.pfb b/test/input/esmx/LW_slopex.pfb similarity index 100% rename from test/esmx/PARFLOW_INPUTS/slopex_LW.pfb rename to test/input/esmx/LW_slopex.pfb diff --git a/test/esmx/PARFLOW_INPUTS/slopey_LW.pfb b/test/input/esmx/LW_slopey.pfb similarity index 100% rename from test/esmx/PARFLOW_INPUTS/slopey_LW.pfb rename to test/input/esmx/LW_slopey.pfb