diff --git a/.github/workflows/CEFI_MOM6-ci._yaml b/.github/workflows/CEFI_MOM6-ci.yaml similarity index 76% rename from .github/workflows/CEFI_MOM6-ci._yaml rename to .github/workflows/CEFI_MOM6-ci.yaml index a5feeb733..f225b22a6 100644 --- a/.github/workflows/CEFI_MOM6-ci._yaml +++ b/.github/workflows/CEFI_MOM6-ci.yaml @@ -1,7 +1,7 @@ name: CEFI-MOM6-ci on: - # Triggers this workflow on pull request event with "CEFI_MOM6_RT_gaea_c5" label + # Triggers this workflow on pull request event with "CEFI_MOM6_RT_container" label pull_request: branches: [ "main" ] types: [ labeled ] @@ -14,8 +14,8 @@ env: # jobs: checkout-build: - if: ${{ github.event.label.name == 'CEFI_MOM6_RT_gaea_c5' }} - runs-on: self-hosted + if: ${{ github.event.label.name == 'CEFI_MOM6_RT_container' }} + runs-on: [self-hosted, container] timeout-minutes: 600 strategy: max-parallel: 1 @@ -28,9 +28,12 @@ jobs: fetch-depth: 1 submodules: recursive - - name: Build MOM6SIS2 + - name: Build MOM6SIS2 using container run: | cd ${{ env.TEST_DIR }}/CEFI_MOM6_CHECK/builds + pwd + #img=/gpfs/f6/ira-cefi/world-shared/container/ubuntu22.04-intel-ufs-env-v1.8.0-llvm.img + #apptainer exec -B /gpfs -B /ncrc/home2/Yi-cheng.Teng:/ncrc/home2/Yi-cheng.Teng $img bash linux-build.bash -m docker -p linux-intel -t repro -f mom6sis2 jobid=$(sbatch --parsable ci_build_driver.sh | awk -F';' '{print $1}' | cut -f1) # sleep 1 @@ -45,8 +48,8 @@ jobs: fi sleep 60 # Adjust the sleep duration as needed done - # - check_file="${{ env.TEST_DIR }}/CEFI_MOM6_CHECK/builds/build/gaea-ncrc5.intel23/ocean_ice/repro/MOM6SIS2" + + check_file="${{ env.TEST_DIR }}/CEFI_MOM6_CHECK/builds/build/docker-linux-intel/ocean_ice/repro/MOM6SIS2" if [ -f "$check_file" ]; then echo "PASSED: $check_file" else @@ -56,16 +59,16 @@ jobs: run-CEFI_MOM6-ci: needs: checkout-build - runs-on: self-hosted + runs-on: [self-hosted, container] strategy: - max-parallel: 2 + max-parallel: 1 matrix: - case: ["NWA12.COBALT", "NEP10.COBALT"] + case: ["NEP10.COBALT"] steps: - name: Run Experiment ${{ matrix.case }} run: | cd ${{ env.TEST_DIR }}/CEFI_MOM6_CHECK/exps/${{ matrix.case }} - jobid=$(sbatch --parsable driver.sh | awk -F';' '{print $1}' | cut -f1) + jobid=$(sbatch --parsable run_container.sh | awk -F';' '{print $1}' | cut -f1) # sleep 1 while :; do @@ -80,7 +83,7 @@ jobs: sleep 60 # Adjust the sleep duration as needed done # - expected_string="All restart files are identical, PASS" + expected_string="ocean.stats is identical to ref, PASS" check_file="${{ env.TEST_DIR }}/CEFI_MOM6_CHECK/exps/${{ matrix.case }}/${{ matrix.case }}_o.$jobid" if [ -f "$check_file" ]; then if grep -qF "$expected_string" $check_file; then @@ -96,22 +99,22 @@ jobs: add-pass-label: needs: run-CEFI_MOM6-ci - runs-on: self-hosted + runs-on: [self-hosted, container] if: ${{ needs.run-CEFI_MOM6-ci.result == 'success' }} steps: - - name: Add "pass_CEFI_MOM6_RT" label on success + - name: Add "pass_CEFI_MOM6_RT_container" label on success run: | TOKEN=${{ secrets.GITHUB_TOKEN }} - RT_TEST_LABEL="CEFI_MOM6_RT_gaea_c5" - PASS_LABEL="pass_CEFI_MOM6_RT" + RT_TEST_LABEL="CEFI_MOM6_RT_container" + PASS_LABEL="pass_CEFI_MOM6_RT_container" - # Remove the "CEFI_MOM6_RT_gaea_c5" label + # Remove the "CEFI_MOM6_RT_container" label curl -X DELETE \ -H "Authorization: Bearer $TOKEN" \ -H "Accept: application/vnd.github.v3+json" \ "https://api.github.com/repos/$GITHUB_REPOSITORY/issues/${{ env.PR_NUMBER }}/labels/$RT_TEST_LABEL" - # Add the "pass_CEFI_MOM6_RT" label + # Add the "pass_CEFI_MOM6_RT_container" label curl -X POST \ -H "Authorization: Bearer $TOKEN" \ -H "Accept: application/vnd.github.v3+json" \ @@ -120,7 +123,7 @@ jobs: clean-up: needs: add-pass-label - runs-on: self-hosted + runs-on: [self-hosted, container] strategy: max-parallel: 1 steps: diff --git a/builds/ci_build_driver.sh b/builds/ci_build_driver.sh index 5c61a25cb..4f1ea7d53 100755 --- a/builds/ci_build_driver.sh +++ b/builds/ci_build_driver.sh @@ -1,65 +1,23 @@ #!/bin/bash #SBATCH --nodes=1 -#SBATCH --time=60 -#SBATCH --job-name="MOM6SIS2_ci_build" -#SBATCH --output=MOM6SIS2_ci_build_o.%j -#SBATCH --error=MOM6SIS2_ci_build_e.%j -#SBATCH --qos=debug +#SBATCH --time=360 +#SBATCH --job-name="MOM6SIS2_container_build" +#SBATCH --output=MOM6SIS2_container_build_o.%j +#SBATCH --error=MOM6SIS2_container_build_e.%j +#SBATCH --qos=normal #SBATCH --partition=batch -#SBATCH --clusters=c5 -#SBATCH --account=cefi +#SBATCH --clusters=c6 +#SBATCH --account=ira-cefi # [ -d "build" ] && rm -rf build # -echo "Build MOM6SIS2-COBALT for CI testing started: " `date` +echo "Build MOM6SIS2-COBALT using container started: " `date` # -machine_name="gaea" -platform="ncrc5.intel23" -target="repro" -flavor="fms1_mom6sis2" - -FMSlib_PATH="/gpfs/f5/cefi/scratch/Yi-cheng.Teng/github/FMS/2024.02_FMS1" -rootdir=$(pwd) -abs_rootdir=$rootdir - -echo $abs_rootdir - -#load modules -source $MODULESHOME/init/bash -source $rootdir/$machine_name/$platform.env -. $rootdir/$machine_name/$platform.env - -makeflags="NETCDF=3" - -if [[ $target =~ "repro" ]] ; then - makeflags="$makeflags REPRO=1" -fi - -srcdir=$abs_rootdir/../src - -# -sed -i 's/static pid_t gettid(void)/pid_t gettid(void)/g' $srcdir/FMS/affinity/affinity.c - -# -if [[ $flavor == "fms1_mom6sis2" ]] ; then - echo "build mom6sis2 with FMS1 cap" - - mkdir -p build/$machine_name-$platform/ocean_ice/$target - pushd build/$machine_name-$platform/ocean_ice/$target - rm -f path_names - $srcdir/mkmf/bin/list_paths $srcdir/MOM6/{config_src/infra/FMS1,config_src/memory/dynamic_symmetric,config_src/drivers/FMS_cap,config_src/external/ODA_hooks,config_src/external/database_comms,config_src/external/drifters,config_src/external/stochastic_physics,pkg/GSW-Fortran/{modules,toolbox}/,src/{*,*/*}/} $srcdir/SIS2/{config_src/dynamic_symmetric,config_src/external/Icepack_interfaces,src} $srcdir/icebergs/src $srcdir/FMS/{coupler,include}/ $srcdir/{ocean_BGC/generic_tracers,ocean_BGC/mocsy/src}/ $srcdir/{atmos_null,ice_param,land_null,coupler/shared/,coupler/full/}/ - - compiler_options='-DINTERNAL_FILE_NML -DMAX_FIELDS_=600 -DNOT_SET_AFFINITY -Duse_deprecated_io -D_USE_MOM6_DIAG -D_USE_GENERIC_TRACER -DUSE_PRECISION=2 -D_USE_LEGACY_LAND_ -Duse_AM3_physics' - - $srcdir/mkmf/bin/mkmf -t $abs_rootdir/$machine_name/$platform.mk -o "-I${FMSlib_PATH}/shared/$target" -p MOM6SIS2 -l "-L${FMSlib_PATH}/shared/$target -lfms" -c "$compiler_options" path_names - - make $makeflags MOM6SIS2 - -fi - +export img=/gpfs/f6/ira-cefi/world-shared/container/ubuntu22.04-intel-ufs-env-v1.8.0-llvm.img +apptainer exec -B /gpfs -B /ncrc/home2/Yi-cheng.Teng:/ncrc/home2/Yi-cheng.Teng $img bash linux-build.bash -m docker -p linux-intel -t repro -f mom6sis2 # -echo "Build MOM6SIS2-COBALT for CI testing ended: " `date` +echo "Build MOM6SIS2-COBALT using container ended: " `date` diff --git a/builds/container-scripts/externalize.sh b/builds/container-scripts/externalize.sh new file mode 100755 index 000000000..b87eda967 --- /dev/null +++ b/builds/container-scripts/externalize.sh @@ -0,0 +1,110 @@ +#!/bin/bash + +################################################################################ +# Help # +################################################################################ +Help() +{ + # Display Help + echo "Add description of the script functions here." + echo + echo "Syntax: scriptTemplate [-|h|e|p]" + echo "options:" + echo "-h Print this Help." + echo "-e Create the external executable in the user-specified folder" + echo "-p env file that contains the necessary modules within the container" + echo +} + +################################################################################ +################################################################################ +# Main program # +################################################################################ +################################################################################ +################################################################################ +# Process the input options. Add options as needed. # +################################################################################ +# Get the options +#while getopts ":hep" option; do +# case $option in +# h) # display Help +# Help +# exit;; +# e) # external directory to hold externalized executables +# exec_dir=$2 +# echo "Will create external executable in $exec_dir" +# p) # env file that contains the necessary modules inside the container +# env_file=$3 +# echo "Will load modules in $env_file" +# +# esac +#done +#shift $(($OPTIND )) + +while getopts ":he:p:" option; do + case $option in + h) # display Help + Help + exit + ;; + e) # external directory to hold externalized executables + exec_dir=$OPTARG + echo "Will create external executable in $exec_dir" + ;; + p) # env file that contains the necessary modules inside the container + env_file=$OPTARG + echo "Will load modules in $env_file" + ;; + esac +done + +# Shift past the processed options +shift $((OPTIND -1)) + + +fileList=$@ + +source $env_file +mkdir -p $exec_dir +cp /opt/container-scripts/run_container_executable.sh $exec_dir +cp /opt/container-scripts/build_container_executable.sh $exec_dir +#replace the paths in the script +sed -i "s|IMAGE|$SINGULARITY_CONTAINER|g" $exec_dir/*_executable.sh +nbinds=`echo $SINGULARITY_BIND | awk -F "," '{print NF }'` +bindstring=" " +for (( i = 1; i <= $nbinds; i++ )); do binddir=`echo $SINGULARITY_BIND | cut -d "," -f $i` && bindstring="${bindstring} -B ${binddir}" ; done +echo $bindstring +sed -i "s|BINDDIRS|$bindstring|g" $exec_dir/*_executable.sh +sed -i "s|LDLIB_PATH|$LD_LIBRARY_PATH|g" $exec_dir/*_executable.sh +sed -i "s|LIB_PATH|$LIBRARY_PATH|g" $exec_dir/*_executable.sh +sed -i "s|FI_PATH|$FI_PROVIDER_PATH|g" $exec_dir/*_executable.sh + +for file in $fileList +do + fullfile=$(readlink -m $file) + basefile=$(basename "$fullfile") + cp $exec_dir/run_container_executable.sh $exec_dir/$basefile + pathdir=$(dirname $fullfile) + echo "fullfile is $fullfile" + echo $pathdir + + EXEC_PATH="$pathdir:$PATH" + sed -i "s|EXEC_PATH|$EXEC_PATH|g" $exec_dir/$basefile + sed -i "s|ESMF_MK|$ESMFMKFILE|g" $exec_dir/$basefile +done +#fileList="make cmake ecbuild python python3" +fileList="" +for file in $fileList +do + fullfile=$(which $file) + basefile=$(basename "$fullfile") + cp $exec_dir/build_container_executable.sh $exec_dir/$basefile + pathdir=$(dirname $fullfile) + + EXEC_PATH="$pathdir:$PATH" + sed -i "s|EXEC_PATH|$EXEC_PATH|g" $exec_dir/$basefile + sed -i "s|CMAKE_PREPATH|$CMAKE_PREFIX_PATH|g" $exec_dir/$basefile + sed -i "s|ESMF_MK|$ESMFMKFILE|g" $exec_dir/$basefile +done + +chmod +x $exec_dir/* diff --git a/builds/docker/linux-intel.env b/builds/docker/linux-intel.env new file mode 100644 index 000000000..2e93a192c --- /dev/null +++ b/builds/docker/linux-intel.env @@ -0,0 +1,7 @@ +source /usr/lmod/lmod/init/bash +module purge +module use /opt/spack-stack/spack-stack-1.8.0/envs/unified-env/install/modulefiles/Core +module load stack-oneapi +module load stack-intel-oneapi-mpi +module load jedi-ufs-env +module unload fms/2024.02 diff --git a/builds/docker/linux-intel.mk b/builds/docker/linux-intel.mk new file mode 100644 index 000000000..c21daa1e2 --- /dev/null +++ b/builds/docker/linux-intel.mk @@ -0,0 +1,280 @@ +# Template for the Intel Compilers on NASA NCCS System +# +# Typical use with mkmf +# mkmf -t nccs-intel.mk -c"-Duse_libMPI -Duse_netCDF" path_names /usr/local/include + +############ +# Commands Macros +############ +FC = mpiifx +CC = mpiicx +LD = mpiifx + +####################### +# Build target macros +# +# Macros that modify compiler flags used in the build. Target +# macrose are usually set on the call to make: +# +# make REPRO=on NETCDF=3 +# +# Most target macros are activated when their value is non-blank. +# Some have a single value that is checked. Others will use the +# value of the macro in the compile command. + +DEBUG = # If non-blank, perform a debug build (Cannot be + # mixed with REPRO or TEST) + +REPRO = # If non-blank, erform a build that guarentees + # reprodicuibilty from run to run. Cannot be used + # with DEBUG or TEST + +TEST = # If non-blank, use the compiler options defined in + # the FFLAGS_TEST and CFLAGS_TEST macros. Cannot be + # use with REPRO or DEBUG + +VERBOSE = # If non-blank, add additional verbosity compiler + # options + +OPENMP = # If non-blank, compile with openmp enabled + +NO_OVERRIDE_LIMITS = # If non-blank, do not use the -qoverride-limits + # compiler option. Default behavior is to compile + # with -qoverride-limits. + +NETCDF = # If value is '3' and CPPDEFS contains + # '-Duse_netCDF', then the additional cpp macro + # '-Duse_LARGEFILE' is added to the CPPDEFS macro. + +INCLUDES = # A list of -I Include directories to be added to the + # the compile command. + +SSE = -march=core-avx2 # The SSE options to be used to compile. If blank, + # than use the default SSE settings for the host. + # Current default is to use SSE2. + +COVERAGE = # Add the code coverage compile options. + +# Need to use at least GNU Make version 3.81 +need := 3.81 +ok := $(filter $(need),$(firstword $(sort $(MAKE_VERSION) $(need)))) +ifneq ($(need),$(ok)) +$(error Need at least make version $(need). Load module gmake/3.81) +endif + +# REPRO, DEBUG and TEST need to be mutually exclusive of each other. +# Make sure the user hasn't supplied two at the same time +ifdef REPRO +ifneq ($(DEBUG),) +$(error Options REPRO and DEBUG cannot be used together) +else ifneq ($(TEST),) +$(error Options REPRO and TEST cannot be used together) +endif +else ifdef DEBUG +ifneq ($(TEST),) +$(error Options DEBUG and TEST cannot be used together) +endif +endif + +# Required Preprocessor Macros: +CPPDEFS += -Duse_netCDF + +# Additional Preprocessor Macros needed due to Autotools and CMake +CPPDEFS += -DHAVE_SCHED_GETAFFINITY + +# Macro for Fortran preprocessor +FPPFLAGS = -fpp -Wp,-w $(INCLUDES) +# Fortran Compiler flags for the NetCDF library +FPPFLAGS += $(shell nf-config --fflags) + +# Base set of Fortran compiler flags +FFLAGS := -fno-alias -auto -safe-cray-ptr -ftz -assume byterecl -i4 -r8 -nowarn -traceback +#FFLAGS := -fno-alias -auto -safe-cray-ptr -ftz -assume byterecl -i4 -r8 -traceback + +# Flags based on perforance target (production (OPT), reproduction (REPRO), or debug (DEBUG) +FFLAGS_OPT = -O3 -debug minimal -fp-model precise +FFLAGS_REPRO = -O2 -debug minimal -fp-model precise +#FFLAGS_DEBUG = -g -O0 -check -check noarg_temp_created -check nopointer -warn -warn noerrors -fpe0 -ftrapuv +FFLAGS_DEBUG = -g -O0 -check -check nouninit -check noarg_temp_created -check nopointer -fpe0 -ftrapuv -init=snan,arrays + +# Flags to add additional build options +FFLAGS_OPENMP = -qopenmp +FFLAGS_OVERRIDE_LIMITS = -qoverride-limits +FFLAGS_VERBOSE = -v -V -what -warn all +FFLAGS_COVERAGE = -prof-gen=srcpos + +# Macro for C preprocessor +CPPFLAGS = -D__IFC $(INCLUDES) +# C Compiler flags for the NetCDF library +CPPFLAGS += $(shell nc-config --cflags) + +# Base set of C compiler flags +CFLAGS := -traceback + +# Flags based on perforance target (production (OPT), reproduction (REPRO), or debug (DEBUG) +CFLAGS_OPT = -O2 -debug minimal +CFLAGS_REPRO = -O2 -debug minimal +CFLAGS_DEBUG = -O0 -g -ftrapuv + +# Flags to add additional build options +CFLAGS_OPENMP = -qopenmp +CFLAGS_VERBOSE = -w3 +CFLAGS_COVERAGE = -prof-gen=srcpos + +# Optional Testing compile flags. Mutually exclusive from DEBUG, REPRO, and OPT +# *_TEST will match the production if no new option(s) is(are) to be tested. +FFLAGS_TEST = $(FFLAGS_OPT) +CFLAGS_TEST = $(CFLAGS_OPT) + +# Linking flags +LDFLAGS = $(shell nc-config --libs) +LDFLAGS += $(shell nf-config --flibs) +LDFLAGS_OPENMP = -qopenmp +LDFLAGS_VERBOSE = -Wl,-V,--verbose,-cref,-M +LDFLAGS_COVERAGE = -prof-gen=srcpos + +# Start with blank LIBS +LIBS = + +# Get compile flags based on target macros. +ifdef REPRO +CFLAGS += $(CFLAGS_REPRO) +FFLAGS += $(FFLAGS_REPRO) +else ifdef DEBUG +CFLAGS += $(CFLAGS_DEBUG) +FFLAGS += $(FFLAGS_DEBUG) +else ifdef TEST +CFLAGS += $(CFLAGS_TEST) +FFLAGS += $(FFLAGS_TEST) +else +CFLAGS += $(CFLAGS_OPT) +FFLAGS += $(FFLAGS_OPT) +endif + +ifdef OPENMP +CFLAGS += $(CFLAGS_OPENMP) +FFLAGS += $(FFLAGS_OPENMP) +LDFLAGS += $(LDFLAGS_OPENMP) +endif + +ifdef SSE +CFLAGS += $(SSE) +FFLAGS += $(SSE) +endif + +ifdef NO_OVERRIDE_LIMITS +FFLAGS += $(FFLAGS_OVERRIDE_LIMITS) +endif + +ifdef VERBOSE +CFLAGS += $(CFLAGS_VERBOSE) +FFLAGS += $(FFLAGS_VERBOSE) +LDFLAGS += $(LDFLAGS_VERBOSE) +endif + +ifeq ($(NETCDF),3) + # add the use_LARGEFILE cppdef + ifneq ($(findstring -Duse_netCDF,$(CPPDEFS)),) + CPPDEFS += -Duse_LARGEFILE + endif +endif + +ifdef COVERAGE +ifdef BUILDROOT +PROF_DIR=-prof-dir=$(BUILDROOT) +endif +CFLAGS += $(CFLAGS_COVERAGE) $(PROF_DIR) +FFLAGS += $(FFLAGS_COVERAGE) $(PROF_DIR) +LDFLAGS += $(LDFLAGS_COVERAGE) $(PROF_DIR) +endif + +LDFLAGS += $(LIBS) + +#--------------------------------------------------------------------------- +# you should never need to change any lines below. + +# see the MIPSPro F90 manual for more details on some of the file extensions +# discussed here. +# this makefile template recognizes fortran sourcefiles with extensions +# .f, .f90, .F, .F90. Given a sourcefile ., where is one of +# the above, this provides a number of default actions: + +# make .opt create an optimization report +# make .o create an object file +# make .s create an assembly listing +# make .x create an executable file, assuming standalone +# source +# make .i create a preprocessed file (for .F) +# make .i90 create a preprocessed file (for .F90) + +# The macro TMPFILES is provided to slate files like the above for removal. + +RM = rm -f +TMPFILES = .*.m *.B *.L *.i *.i90 *.l *.s *.mod *.opt + +.SUFFIXES: .F .F90 .H .L .T .f .f90 .h .i .i90 .l .o .s .opt .x + +.f.L: + $(FC) $(FFLAGS) -c -listing $*.f +.f.opt: + $(FC) $(FFLAGS) -c -opt_report_level max -opt_report_phase all -opt_report_file $*.opt $*.f +.f.l: + $(FC) $(FFLAGS) -c $(LIST) $*.f +.f.T: + $(FC) $(FFLAGS) -c -cif $*.f +.f.o: + $(FC) $(FFLAGS) -c $*.f +.f.s: + $(FC) $(FFLAGS) -S $*.f +.f.x: + $(FC) $(FFLAGS) -o $*.x $*.f *.o $(LDFLAGS) +.f90.L: + $(FC) $(FFLAGS) -c -listing $*.f90 +.f90.opt: + $(FC) $(FFLAGS) -c -opt_report_level max -opt_report_phase all -opt_report_file $*.opt $*.f90 +.f90.l: + $(FC) $(FFLAGS) -c $(LIST) $*.f90 +.f90.T: + $(FC) $(FFLAGS) -c -cif $*.f90 +.f90.o: + $(FC) $(FFLAGS) -c $*.f90 +.f90.s: + $(FC) $(FFLAGS) -c -S $*.f90 +.f90.x: + $(FC) $(FFLAGS) -o $*.x $*.f90 *.o $(LDFLAGS) +.F.L: + $(FC) $(CPPDEFS) $(FPPFLAGS) $(FFLAGS) -c -listing $*.F +.F.opt: + $(FC) $(CPPDEFS) $(FPPFLAGS) $(FFLAGS) -c -opt_report_level max -opt_report_phase all -opt_report_file $*.opt $*.F +.F.l: + $(FC) $(CPPDEFS) $(FPPFLAGS) $(FFLAGS) -c $(LIST) $*.F +.F.T: + $(FC) $(CPPDEFS) $(FPPFLAGS) $(FFLAGS) -c -cif $*.F +.F.f: + $(FC) $(CPPDEFS) $(FPPFLAGS) -EP $*.F > $*.f +.F.i: + $(FC) $(CPPDEFS) $(FPPFLAGS) -P $*.F +.F.o: + $(FC) $(CPPDEFS) $(FPPFLAGS) $(FFLAGS) -c $*.F +.F.s: + $(FC) $(CPPDEFS) $(FPPFLAGS) $(FFLAGS) -c -S $*.F +.F.x: + $(FC) $(CPPDEFS) $(FPPFLAGS) $(FFLAGS) -o $*.x $*.F *.o $(LDFLAGS) +.F90.L: + $(FC) $(CPPDEFS) $(FPPFLAGS) $(FFLAGS) -c -listing $*.F90 +.F90.opt: + $(FC) $(CPPDEFS) $(FPPFLAGS) $(FFLAGS) -c -opt_report_level max -opt_report_phase all -opt_report_file $*.opt $*.F90 +.F90.l: + $(FC) $(CPPDEFS) $(FPPFLAGS) $(FFLAGS) -c $(LIST) $*.F90 +.F90.T: + $(FC) $(CPPDEFS) $(FPPFLAGS) $(FFLAGS) -c -cif $*.F90 +.F90.f90: + $(FC) $(CPPDEFS) $(FPPFLAGS) -EP $*.F90 > $*.f90 +.F90.i90: + $(FC) $(CPPDEFS) $(FPPFLAGS) -P $*.F90 +.F90.o: + $(FC) $(CPPDEFS) $(FPPFLAGS) $(FFLAGS) -c $*.F90 +.F90.s: + $(FC) $(CPPDEFS) $(FPPFLAGS) $(FFLAGS) -c -S $*.F90 +.F90.x: + $(FC) $(CPPDEFS) $(FPPFLAGS) $(FFLAGS) -o $*.x $*.F90 *.o $(LDFLAGS) diff --git a/builds/linux-build.bash b/builds/linux-build.bash index 481492bae..797b29a77 100755 --- a/builds/linux-build.bash +++ b/builds/linux-build.bash @@ -44,7 +44,7 @@ abs_rootdir=`cd $rootdir && pwd` #load modules source $MODULESHOME/init/bash source $rootdir/$machine_name/$platform.env -. $rootdir/$machine_name/$platform.env +#. $rootdir/$machine_name/$platform.env makeflags="NETCDF=3" diff --git a/exps/NEP10.COBALT/ref/docker-linux-intel-repro/ocean.stats b/exps/NEP10.COBALT/ref/docker-linux-intel-repro/ocean.stats new file mode 100644 index 000000000..68669e7c3 --- /dev/null +++ b/exps/NEP10.COBALT/ref/docker-linux-intel-repro/ocean.stats @@ -0,0 +1,4 @@ + Step, Day, Truncs, Energy/Mass, Maximum CFL, Mean Sea Level, Total Mass, Mean Salin, Mean Temp, Frac Mass Err, Salin Err, Temp Err + [days] [m2 s-2] [Nondim] [m] [kg] [PSU] [degC] [Nondim] [PSU] [degC] + 0, 727563.000, 0, En 9.8480593715804025E-04, CFL 0.03314, SL 0.0000E+00, M 7.10037E+19, S 34.4969, T 2.7277, Me 0.00E+00, Se 0.00E+00, Te 0.00E+00 + 24, 727564.000, 0, En 1.1780194089218764E-03, CFL 0.16802, SL 0.0000E+00, M 7.10012E+19, S 34.4969, T 2.7272, Me -3.48E-05, Se -1.21E-03, Te 9.86E-05 diff --git a/exps/NEP10.COBALT/run_container.sh b/exps/NEP10.COBALT/run_container.sh new file mode 100644 index 000000000..93aafaca9 --- /dev/null +++ b/exps/NEP10.COBALT/run_container.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +#SBATCH --nodes=5 +#SBATCH --time=60 +#SBATCH --job-name="container_NEP10" +#SBATCH --output=NEP10.COBALT_o.%j +#SBATCH --error=NEP10.COBALT_e.%j +#SBATCH --qos=debug +#SBATCH --partition=batch +#SBATCH --clusters=c6 +#SBATCH --account=ira-cefi + +echo "Model started: " `date` + +# +echo "link datasets ..." +pushd ../ +ln -fs /gpfs/f6/ira-cefi/world-shared/datasets ./ +popd + + +# +ln -fs input.nml_24hr input.nml + +# +ln -fs /gpfs/f6/ira-cefi/proj-shared/github/tmp/NEP10/RESTART_container ./RESTART +rm /gpfs/f6/ira-cefi/proj-shared/github/tmp/NEP10/RESTART_container/* + +# +export img=/gpfs/f6/ira-cefi/world-shared/container/ubuntu22.04-intel-ufs-env-v1.8.0-llvm.img +apptainer exec -B /gpfs -B /ncrc/home2/Yi-cheng.Teng:/ncrc/home2/Yi-cheng.Teng $img bash ../../builds/container-scripts/externalize.sh -e container_exec -p ../../builds/docker/linux-intel.env ../../builds/build/docker-linux-intel/ocean_ice/repro/MOM6SIS2 + +# Avoid job errors because of filesystem synchronization delays +sync && sleep 1 +srun --mpi=pmi2 --ntasks=904 --cpus-per-task=1 --export=ALL ./container_exec/MOM6SIS2 + +echo "Model ended: " `date` + +# +diff -q ./ocean.stats ref/docker-linux-intel-repro/ocean.stats > /dev/null || { echo "Error: ocean.stats is not identical to ref, please check! Exiting now..."; exit 1; } +echo "ocean.stats is identical to ref, PASS" + +# +rm -rf /gpfs/f6/ira-cefi/proj-shared/github/tmp/NEP10/RESTART_container/* diff --git a/exps/test/run_container.sh b/exps/test/run_container.sh new file mode 100644 index 000000000..1f82cf1e8 --- /dev/null +++ b/exps/test/run_container.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +#SBATCH --nodes=5 +#SBATCH --time=60 +#SBATCH --job-name="container_NEP10" +#SBATCH --output=NEP10.COBALT_o.%j +#SBATCH --error=NEP10.COBALT_e.%j +#SBATCH --qos=debug +#SBATCH --partition=batch +#SBATCH --clusters=c6 +#SBATCH --account=ira-cefi + +echo "Model started: " `date` + +# +ln -fs input.nml_24hr input.nml + +# +ln -fs /gpfs/f6/ira-cefi/proj-shared/github/tmp/NEP10/RESTART_container ./RESTART +rm /gpfs/f6/ira-cefi/proj-shared/github/tmp/NEP10/RESTART_container/* + +# +export img=/gpfs/f6/ira-cefi/world-shared/container/ubuntu22.04-intel-ufs-env-v1.8.0-llvm.img +apptainer exec -B /gpfs $img bash ../../builds/container-scripts/externalize.sh -e container_exec -p ../../builds/docker/linux-intel.env ../../builds/build/docker-linux-intel/ocean_ice/repro/MOM6SIS2 + +# Avoid job errors because of filesystem synchronization delays +sync && sleep 1 +srun --mpi=pmi2 --ntasks=904 --cpus-per-task=1 --export=ALL ./container_exec/MOM6SIS2 + +echo "Model ended: " `date` + +# +diff -q ./ocean.stats ref/docker-linux-intel-repro/ocean.stats > /dev/null || { echo "Error: ocean.stats is not identical to ref, please check! Exiting now..."; exit 1; } +echo "ocean.stats is identical to ref, PASS" + +# +rm -rf /gpfs/f6/ira-cefi/proj-shared/github/tmp/NEP10/RESTART_container/*