diff --git a/.github/tools/install-intel-oneapi.sh b/.github/tools/install-intel-oneapi.sh index 78af1aec0..b6db3853b 100755 --- a/.github/tools/install-intel-oneapi.sh +++ b/.github/tools/install-intel-oneapi.sh @@ -1,14 +1,14 @@ -#!/bin/sh +#!/usr/bin/env bash -KEY=GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB +version=2023.2.0 +KEY=GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB wget https://apt.repos.intel.com/intel-gpg-keys/$KEY sudo apt-key add $KEY rm $KEY echo "deb https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list sudo apt-get update sudo apt-get install \ - intel-oneapi-compiler-fortran \ - intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic \ - intel-oneapi-mpi \ - intel-oneapi-mpi-devel \ - intel-oneapi-mkl + intel-oneapi-compiler-fortran-$version \ + intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic-$version \ + intel-oneapi-mpi-devel-2021.10.0 \ + intel-oneapi-mkl-$version diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 089d70db8..1c7a389e4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -34,8 +34,7 @@ jobs: name: - linux gnu-10 - linux clang-12 - - linux nvhpc-21.9 - - linux intel + - linux intel-classic - macos include: @@ -74,9 +73,9 @@ jobs: cmake_options: -DCMAKE_CXX_FLAGS=--diag_suppress177 caching: true - - name : linux intel + - name : linux intel-classic os: ubuntu-20.04 - compiler: intel-oneapi + compiler: intel-classic compiler_cc: icc compiler_cxx: icpc compiler_fc: ifort @@ -152,7 +151,7 @@ jobs: ${ECWAM_TOOLS}/install-intel-oneapi.sh source /opt/intel/oneapi/setvars.sh printenv >> $GITHUB_ENV - echo "CACHE_SUFFIX=$(icc -dumpversion)" >> $GITHUB_ENV + echo "CACHE_SUFFIX=$CC-$($CC -dumpversion)" >> $GITHUB_ENV - name: Install MPI shell: bash -eux {0} diff --git a/src/ecwam/CMakeLists.txt b/src/ecwam/CMakeLists.txt index d7f308f8a..d6b0dca4c 100644 --- a/src/ecwam/CMakeLists.txt +++ b/src/ecwam/CMakeLists.txt @@ -225,6 +225,7 @@ list( APPEND ecwam_srcs vplus_d.F90 w_maxh.F90 w_mode_st.F90 + wam_init_gpu_mod.F90 wam_multio_mod.F90 wam_nproma.F90 wam_sorti.F90 diff --git a/src/ecwam/ecwam_loki.config b/src/ecwam/ecwam_loki.config index a6a0a40bf..fd95a55b2 100644 --- a/src/ecwam/ecwam_loki.config +++ b/src/ecwam/ecwam_loki.config @@ -3,30 +3,50 @@ mode = "idem" role = "kernel" expand = true strict = true -disable = ["DR_HOOK", "lhook", "jphook", "omp_get_wtime", "flush", "jwim", "jwrb", "jwru", "jwro", "environment", "frequency", "forcing_fields", "intgt_param_fields", "wave2ocean", "abort1", "iu06"] +disable = [ + "DR_HOOK", "lhook", "jphook", "omp_get_wtime", "flush", "jwim", + "jwrb", "jwru", "jwro", "environment", "frequency", + "forcing_fields", "intgt_param_fields", "wave2ocean", "abort1", + "iu06" +] utility_routines = ['dr_hook', 'abort1', 'write(iu06'] enable_imports = false -[[routine]] -name = "wamintgr" -role = "driver" -expand = true -disable = ["PROPAG_WAM", "NEWWIND", "GSTATS", "FREQUENCY_FIELD%INIT", "ENVIRONMENT_FIELD%INIT", "FORCING_FIELDS_FIELD%INIT", "WAVE2OCEAN_FIELD%INIT", "INTGT_PARAM_FIELDS_FIELD%INIT", "FREQUENCY_FIELD%UPDATE_VIEW", "ENVIRONMENT_FIELD%UPDATE_VIEW", "FORCING_FIELDS_FIELD%UPDATE_VIEW", "WAVE2OCEAN_FIELD%UPDATE_VIEW", "INTGT_PARAM_FIELDS_FIELD%UPDATE_VIEW", "SOURCE_CONTRIBS_FIELD%INIT", "SOURCE_CONTRIBS_FIELD%UPDATE_VIEW", "INCDATE", "DR_HOOK", "omp_get_wtime", "abort1", "wam_user_clock", "FREQUENCY_FIELD%FINAL", "ENVIRONMENT_FIELD%FINAL", "FORCING_FIELDS_FIELD%FINAL", "WAVE2OCEAN_FIELD%FINAL", "INTGT_PARAM_FIELDS_FIELD%FINAL", "SOURCE_CONTRIBS_FIELD%INIT", "SOURCE_CONTRIBS_FIELD%FINAL"] -real_kind = 'JWRB' +# Define entry point for call-tree transformation +[routines] -[[dimension]] -name = "horizontal" -size = "KIJL" -index = "IJ" -bounds = ["KIJS", "KIJL"] -aliases = ["NPROMA_WAM"] +[routines.wamintgr] + role = "driver" + expand = true + disable = [ + "PROPAG_WAM", "NEWWIND", "GSTATS", "FREQUENCY_FIELD%INIT", + "ENVIRONMENT_FIELD%INIT", "FORCING_FIELDS_FIELD%INIT", + "WAVE2OCEAN_FIELD%INIT", "INTGT_PARAM_FIELDS_FIELD%INIT", + "FREQUENCY_FIELD%UPDATE_VIEW", "ENVIRONMENT_FIELD%UPDATE_VIEW", + "FORCING_FIELDS_FIELD%UPDATE_VIEW", "WAVE2OCEAN_FIELD%UPDATE_VIEW", + "INTGT_PARAM_FIELDS_FIELD%UPDATE_VIEW", + "SOURCE_CONTRIBS_FIELD%INIT", "SOURCE_CONTRIBS_FIELD%UPDATE_VIEW", + "INCDATE", "DR_HOOK", "omp_get_wtime", "abort1", "wam_user_clock", + "FREQUENCY_FIELD%FINAL", "ENVIRONMENT_FIELD%FINAL", + "FORCING_FIELDS_FIELD%FINAL", "WAVE2OCEAN_FIELD%FINAL", + "INTGT_PARAM_FIELDS_FIELD%FINAL", "SOURCE_CONTRIBS_FIELD%INIT", + "SOURCE_CONTRIBS_FIELD%FINAL" + ] + real_kind = 'JWRB' -[[dimension]] -name = "vertical" -size = "NFRE" -index = "M" +# Define indices and bounds for array dimensions +[dimensions] -[[dimension]] -name = "block_dim" -size = "NCHNK" -index = "ICHNK" +[dimensions.horizontal] + size = "KIJL" + index = "IJ" + bounds = ["KIJS", "KIJL"] + aliases = ["NPROMA_WAM"] + +[dimensions.vertical] + size = "NFRE" + index = "M" + +[dimensions.block_dim] + size = "NCHNK" + index = "ICHNK" diff --git a/src/ecwam/ecwam_loki_gpu.config b/src/ecwam/ecwam_loki_gpu.config index 819b27bcc..302bf9685 100644 --- a/src/ecwam/ecwam_loki_gpu.config +++ b/src/ecwam/ecwam_loki_gpu.config @@ -3,31 +3,56 @@ mode = "idem" role = "kernel" expand = true strict = true -disable = ["DR_HOOK", "lhook", "jphook", "omp_get_wtime", "flush", "jwim", "jwrb", "jwru", "jwro", "environment", "frequency", "forcing_fields", "intgt_param_fields", "wave2ocean", "abort1", "iu06"] +disable = [ + "DR_HOOK", "lhook", "jphook", "omp_get_wtime", "flush", "jwim", + "jwrb", "jwru", "jwro", "environment", "frequency", + "forcing_fields", "intgt_param_fields", "wave2ocean", "abort1", + "iu06" +] utility_routines = ['dr_hook', 'abort1', 'write(iu06'] enable_imports = true -[[routine]] -name = "wamintgr_loki_gpu" -role = "driver" -expand = true -disable = ["PROPAG_WAM", "NEWWIND", "GSTATS", "FREQUENCY_FIELD%INIT", "ENVIRONMENT_FIELD%INIT", "FORCING_FIELDS_FIELD%INIT", "WAVE2OCEAN_FIELD%INIT", "INTGT_PARAM_FIELDS_FIELD%INIT", "FREQUENCY_FIELD%UPDATE_DEVICE", "ENVIRONMENT_FIELD%UPDATE_DEVICE", "FORCING_FIELDS_FIELD%UPDATE_DEVICE", "WAVE2OCEAN_FIELD%UPDATE_DEVICE", "INTGT_PARAM_FIELDS_FIELD%UPDATE_DEVICE", "SOURCE_CONTRIBS_FIELD%INIT", "SOURCE_CONTRIBS_FIELD%UPDATE_DEVICE", "INCDATE", "DR_HOOK", "omp_get_wtime", "abort1", "wam_user_clock", "FREQUENCY_FIELD%ENSURE_HOST", "ENVIRONMENT_FIELD%ENSURE_HOST", "FORCING_FIELDS_FIELD%ENSURE_HOST", "WAVE2OCEAN_FIELD%ENSURE_HOST", "INTGT_PARAM_FIELDS_FIELD%ENSURE_HOST", "SOURCE_CONTRIBS_FIELD%INIT", "SOURCE_CONTRIBS_FIELD%ENSURE_HOST", "FREQUENCY_FIELD%FINAL", "ENVIRONMENT_FIELD%FINAL", "FORCING_FIELDS_FIELD%FINAL", "WAVE2OCEAN_FIELD%FINAL", "INTGT_PARAM_FIELDS_FIELD%FINAL", "SOURCE_CONTRIBS_FIELD%INIT", "SOURCE_CONTRIBS_FIELD%FINAL"] -enable_imports = false -real_kind = 'JWRB' +# Define entry point for call-tree transformation +[routines] -[[dimension]] -name = "horizontal" -size = "KIJL" -index = "IJ" -bounds = ["KIJS", "KIJL"] -aliases = ["NPROMA_WAM"] +[routines.wamintgr_loki_gpu] + role = "driver" + expand = true + disable = [ + "PROPAG_WAM", "NEWWIND", "GSTATS", "FREQUENCY_FIELD%INIT", + "ENVIRONMENT_FIELD%INIT", "FORCING_FIELDS_FIELD%INIT", + "WAVE2OCEAN_FIELD%INIT", "INTGT_PARAM_FIELDS_FIELD%INIT", + "FREQUENCY_FIELD%UPDATE_DEVICE", "ENVIRONMENT_FIELD%UPDATE_DEVICE", + "FORCING_FIELDS_FIELD%UPDATE_DEVICE", + "WAVE2OCEAN_FIELD%UPDATE_DEVICE", + "INTGT_PARAM_FIELDS_FIELD%UPDATE_DEVICE", + "SOURCE_CONTRIBS_FIELD%INIT", "SOURCE_CONTRIBS_FIELD%UPDATE_DEVICE", + "INCDATE", "DR_HOOK", "omp_get_wtime", "abort1", "wam_user_clock", + "FREQUENCY_FIELD%ENSURE_HOST", "ENVIRONMENT_FIELD%ENSURE_HOST", + "FORCING_FIELDS_FIELD%ENSURE_HOST", "WAVE2OCEAN_FIELD%ENSURE_HOST", + "INTGT_PARAM_FIELDS_FIELD%ENSURE_HOST", + "SOURCE_CONTRIBS_FIELD%INIT", "SOURCE_CONTRIBS_FIELD%ENSURE_HOST", + "FREQUENCY_FIELD%FINAL", "ENVIRONMENT_FIELD%FINAL", + "FORCING_FIELDS_FIELD%FINAL", "WAVE2OCEAN_FIELD%FINAL", + "INTGT_PARAM_FIELDS_FIELD%FINAL", "SOURCE_CONTRIBS_FIELD%INIT", + "SOURCE_CONTRIBS_FIELD%FINAL" + ] + enable_imports = false + real_kind = 'JWRB' -[[dimension]] -name = "vertical" -size = "NFRE" -index = "M" +# Define indices and bounds for array dimensions +[dimensions] -[[dimension]] -name = "block_dim" -size = "NCHNK" -index = "ICHNK" +[dimensions.horizontal] + size = "KIJL" + index = "IJ" + bounds = ["KIJS", "KIJL"] + aliases = ["NPROMA_WAM"] + +[dimensions.vertical] + size = "NFRE" + index = "M" + +[dimensions.block_dim] + size = "NCHNK" + index = "ICHNK" diff --git a/src/ecwam/wam_init_gpu_mod.F90 b/src/ecwam/wam_init_gpu_mod.F90 new file mode 100644 index 000000000..2c16e0617 --- /dev/null +++ b/src/ecwam/wam_init_gpu_mod.F90 @@ -0,0 +1,30 @@ +! (C) Copyright 1989- ECMWF. +! +! This software is licensed under the terms of the Apache Licence Version 2.0 +! which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +! In applying this licence, ECMWF does not waive the privileges and immunities +! granted to it by virtue of its status as an intergovernmental organisation +! nor does it submit to any jurisdiction. +! + + MODULE WAM_INIT_GPU_MOD + CONTAINS + SUBROUTINE WAM_INIT_GPU(IRANK) +#ifdef _OPENACC + USE OPENACC +#endif + USE PARKIND_WAVE, ONLY : JWIM + IMPLICIT NONE + + INTEGER(KIND=JWIM), INTENT(IN) :: IRANK + INTEGER :: DEVTYPE, DEVNUM, DEV + + +#ifdef _OPENACC + DEVTYPE = ACC_GET_DEVICE_TYPE() + DEVNUM = ACC_GET_NUM_DEVICES(DEVTYPE) + DEV = MOD(IRANK-1, DEVNUM) + CALL ACC_SET_DEVICE_NUM(DEV, DEVTYPE) +#endif + END SUBROUTINE WAM_INIT_GPU + END MODULE WAM_INIT_GPU_MOD diff --git a/src/ecwam/wvwaminit.F90 b/src/ecwam/wvwaminit.F90 index e2ea50ff0..469409be6 100644 --- a/src/ecwam/wvwaminit.F90 +++ b/src/ecwam/wvwaminit.F90 @@ -33,9 +33,6 @@ SUBROUTINE WVWAMINIT (LLCOUPLED, IULOG, LLRNL, & ! ---------------------------------------------------------------------- -#if defined(WAM_PHYS_GPU) || defined(WAM_CUDA) - USE OPENACC -#endif USE PARKIND_WAVE, ONLY : JWIM, JWRB, JWRU USE YOWMAP , ONLY : AMOSOP ,AMONOP ,IQGAUSS @@ -51,6 +48,7 @@ SUBROUTINE WVWAMINIT (LLCOUPLED, IULOG, LLRNL, & USE MPL_MODULE, ONLY : MPL_MYRANK, MPL_NPROC, MPL_BARRIER USE YOMHOOK , ONLY : LHOOK, DR_HOOK, JPHOOK + USE WAM_INIT_GPU_MOD, ONLY : WAM_INIT_GPU ! ---------------------------------------------------------------------- IMPLICIT NONE @@ -77,7 +75,6 @@ SUBROUTINE WVWAMINIT (LLCOUPLED, IULOG, LLRNL, & LOGICAL, INTENT(IN) :: LLCOUPLED LOGICAL :: LLEXIST LOGICAL, SAVE :: LFRST - INTEGER :: DEVTYPE, DEVNUM, DEV DATA LFRST /.TRUE./ @@ -91,10 +88,7 @@ SUBROUTINE WVWAMINIT (LLCOUPLED, IULOG, LLRNL, & NPROC = MPL_NPROC() #if defined(WAM_PHYS_GPU) || defined(WAM_CUDA) - DEVTYPE = ACC_GET_DEVICE_TYPE() - DEVNUM = ACC_GET_NUM_DEVICES(DEVTYPE) - DEV = MOD(IRANK-1, DEVNUM) - CALL ACC_SET_DEVICE_NUM(DEV, DEVTYPE) + CALL WAM_INIT_GPU(IRANK) #endif ! STANDARD OUTPUT UNIT