Merge pull request #7 from ecmwf-ifs/naml-phys-gpu-loki-conf-update

Loki: Update loki config files to new format
ecmwf-ifs · Dec 19, 2023 · 8442fc4 · 8442fc4
2 parents 4981725 + a01ba3c
commit 8442fc4
Show file tree

Hide file tree

Showing 7 changed files with 132 additions and 63 deletions.
diff --git a/.github/tools/install-intel-oneapi.sh b/.github/tools/install-intel-oneapi.sh
@@ -1,14 +1,14 @@
-#!/bin/sh
+#!/usr/bin/env bash
 
-KEY=GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB
+version=2023.2.0
+KEY=GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB
 wget https://apt.repos.intel.com/intel-gpg-keys/$KEY
 sudo apt-key add $KEY
 rm $KEY
 echo "deb https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list
 sudo apt-get update
 sudo apt-get install \
-    intel-oneapi-compiler-fortran \
-    intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic \
-    intel-oneapi-mpi \
-    intel-oneapi-mpi-devel \
-    intel-oneapi-mkl
+    intel-oneapi-compiler-fortran-$version \
+    intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic-$version \
+    intel-oneapi-mpi-devel-2021.10.0 \
+    intel-oneapi-mkl-$version
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -34,8 +34,7 @@ jobs:
         name:
           - linux gnu-10
           - linux clang-12
-          - linux nvhpc-21.9
-          - linux intel
+          - linux intel-classic
           - macos
 
         include:
@@ -74,9 +73,9 @@ jobs:
             cmake_options: -DCMAKE_CXX_FLAGS=--diag_suppress177
             caching: true
 
-          - name : linux intel
+          - name : linux intel-classic
             os: ubuntu-20.04
-            compiler: intel-oneapi
+            compiler: intel-classic
             compiler_cc: icc
             compiler_cxx: icpc
             compiler_fc: ifort
@@ -152,7 +151,7 @@ jobs:
         ${ECWAM_TOOLS}/install-intel-oneapi.sh
         source /opt/intel/oneapi/setvars.sh
         printenv >> $GITHUB_ENV
-        echo "CACHE_SUFFIX=$(icc -dumpversion)" >> $GITHUB_ENV
+        echo "CACHE_SUFFIX=$CC-$($CC -dumpversion)" >> $GITHUB_ENV
 
     - name: Install MPI
       shell: bash -eux {0}

diff --git a/src/ecwam/CMakeLists.txt b/src/ecwam/CMakeLists.txt
@@ -225,6 +225,7 @@ list( APPEND ecwam_srcs
   vplus_d.F90
   w_maxh.F90
   w_mode_st.F90
+  wam_init_gpu_mod.F90
   wam_multio_mod.F90
   wam_nproma.F90
   wam_sorti.F90

diff --git a/src/ecwam/ecwam_loki.config b/src/ecwam/ecwam_loki.config
@@ -3,30 +3,50 @@ mode = "idem"
 role = "kernel"
 expand =  true
 strict =  true
-disable = ["DR_HOOK", "lhook", "jphook", "omp_get_wtime", "flush", "jwim", "jwrb", "jwru", "jwro", "environment", "frequency", "forcing_fields", "intgt_param_fields", "wave2ocean", "abort1", "iu06"]
+disable = [
+  "DR_HOOK", "lhook", "jphook", "omp_get_wtime", "flush", "jwim",
+  "jwrb", "jwru", "jwro", "environment", "frequency",
+  "forcing_fields", "intgt_param_fields", "wave2ocean", "abort1",
+  "iu06"
+]
 utility_routines = ['dr_hook', 'abort1', 'write(iu06']
 enable_imports = false
 
-[[routine]]
-name = "wamintgr"
-role = "driver"
-expand = true
-disable = ["PROPAG_WAM", "NEWWIND", "GSTATS", "FREQUENCY_FIELD%INIT", "ENVIRONMENT_FIELD%INIT", "FORCING_FIELDS_FIELD%INIT", "WAVE2OCEAN_FIELD%INIT", "INTGT_PARAM_FIELDS_FIELD%INIT", "FREQUENCY_FIELD%UPDATE_VIEW", "ENVIRONMENT_FIELD%UPDATE_VIEW", "FORCING_FIELDS_FIELD%UPDATE_VIEW", "WAVE2OCEAN_FIELD%UPDATE_VIEW", "INTGT_PARAM_FIELDS_FIELD%UPDATE_VIEW", "SOURCE_CONTRIBS_FIELD%INIT", "SOURCE_CONTRIBS_FIELD%UPDATE_VIEW", "INCDATE", "DR_HOOK", "omp_get_wtime", "abort1", "wam_user_clock", "FREQUENCY_FIELD%FINAL", "ENVIRONMENT_FIELD%FINAL", "FORCING_FIELDS_FIELD%FINAL", "WAVE2OCEAN_FIELD%FINAL", "INTGT_PARAM_FIELDS_FIELD%FINAL", "SOURCE_CONTRIBS_FIELD%INIT", "SOURCE_CONTRIBS_FIELD%FINAL"]
-real_kind = 'JWRB'
+# Define entry point for call-tree transformation
+[routines]
 
-[[dimension]]
-name = "horizontal"
-size = "KIJL"
-index = "IJ"
-bounds = ["KIJS", "KIJL"]
-aliases = ["NPROMA_WAM"]
+[routines.wamintgr]
+  role = "driver"
+  expand = true
+  disable = [
+    "PROPAG_WAM", "NEWWIND", "GSTATS", "FREQUENCY_FIELD%INIT",
+    "ENVIRONMENT_FIELD%INIT", "FORCING_FIELDS_FIELD%INIT",
+    "WAVE2OCEAN_FIELD%INIT", "INTGT_PARAM_FIELDS_FIELD%INIT",
+    "FREQUENCY_FIELD%UPDATE_VIEW", "ENVIRONMENT_FIELD%UPDATE_VIEW",
+    "FORCING_FIELDS_FIELD%UPDATE_VIEW", "WAVE2OCEAN_FIELD%UPDATE_VIEW",
+    "INTGT_PARAM_FIELDS_FIELD%UPDATE_VIEW",
+    "SOURCE_CONTRIBS_FIELD%INIT", "SOURCE_CONTRIBS_FIELD%UPDATE_VIEW",
+    "INCDATE", "DR_HOOK", "omp_get_wtime", "abort1", "wam_user_clock",
+    "FREQUENCY_FIELD%FINAL", "ENVIRONMENT_FIELD%FINAL",
+    "FORCING_FIELDS_FIELD%FINAL", "WAVE2OCEAN_FIELD%FINAL",
+    "INTGT_PARAM_FIELDS_FIELD%FINAL", "SOURCE_CONTRIBS_FIELD%INIT",
+    "SOURCE_CONTRIBS_FIELD%FINAL"
+ ]
+ real_kind = 'JWRB'
 
-[[dimension]]
-name = "vertical"
-size = "NFRE"
-index = "M"
+# Define indices and bounds for array dimensions
+[dimensions]
 
-[[dimension]]
-name = "block_dim"
-size = "NCHNK"
-index = "ICHNK"
+[dimensions.horizontal]
+  size = "KIJL"
+  index = "IJ"
+  bounds = ["KIJS", "KIJL"]
+  aliases = ["NPROMA_WAM"]
+
+[dimensions.vertical]
+  size = "NFRE"
+  index = "M"
+
+[dimensions.block_dim]
+  size = "NCHNK"
+  index = "ICHNK"
diff --git a/src/ecwam/ecwam_loki_gpu.config b/src/ecwam/ecwam_loki_gpu.config
@@ -3,31 +3,56 @@ mode = "idem"
 role = "kernel"
 expand =  true
 strict =  true
-disable = ["DR_HOOK", "lhook", "jphook", "omp_get_wtime", "flush", "jwim", "jwrb", "jwru", "jwro", "environment", "frequency", "forcing_fields", "intgt_param_fields", "wave2ocean", "abort1", "iu06"]
+disable = [
+  "DR_HOOK", "lhook", "jphook", "omp_get_wtime", "flush", "jwim",
+  "jwrb", "jwru", "jwro", "environment", "frequency",
+  "forcing_fields", "intgt_param_fields", "wave2ocean", "abort1",
+  "iu06"
+]
 utility_routines = ['dr_hook', 'abort1', 'write(iu06']
 enable_imports = true
 
-[[routine]]
-name = "wamintgr_loki_gpu"
-role = "driver"
-expand = true
-disable = ["PROPAG_WAM", "NEWWIND", "GSTATS", "FREQUENCY_FIELD%INIT", "ENVIRONMENT_FIELD%INIT", "FORCING_FIELDS_FIELD%INIT", "WAVE2OCEAN_FIELD%INIT", "INTGT_PARAM_FIELDS_FIELD%INIT", "FREQUENCY_FIELD%UPDATE_DEVICE", "ENVIRONMENT_FIELD%UPDATE_DEVICE", "FORCING_FIELDS_FIELD%UPDATE_DEVICE", "WAVE2OCEAN_FIELD%UPDATE_DEVICE", "INTGT_PARAM_FIELDS_FIELD%UPDATE_DEVICE", "SOURCE_CONTRIBS_FIELD%INIT", "SOURCE_CONTRIBS_FIELD%UPDATE_DEVICE", "INCDATE", "DR_HOOK", "omp_get_wtime", "abort1", "wam_user_clock", "FREQUENCY_FIELD%ENSURE_HOST", "ENVIRONMENT_FIELD%ENSURE_HOST", "FORCING_FIELDS_FIELD%ENSURE_HOST", "WAVE2OCEAN_FIELD%ENSURE_HOST", "INTGT_PARAM_FIELDS_FIELD%ENSURE_HOST", "SOURCE_CONTRIBS_FIELD%INIT", "SOURCE_CONTRIBS_FIELD%ENSURE_HOST", "FREQUENCY_FIELD%FINAL", "ENVIRONMENT_FIELD%FINAL", "FORCING_FIELDS_FIELD%FINAL", "WAVE2OCEAN_FIELD%FINAL", "INTGT_PARAM_FIELDS_FIELD%FINAL", "SOURCE_CONTRIBS_FIELD%INIT", "SOURCE_CONTRIBS_FIELD%FINAL"]
-enable_imports = false
-real_kind = 'JWRB'
+# Define entry point for call-tree transformation
+[routines]
 
-[[dimension]]
-name = "horizontal"
-size = "KIJL"
-index = "IJ"
-bounds = ["KIJS", "KIJL"]
-aliases = ["NPROMA_WAM"]
+[routines.wamintgr_loki_gpu]
+  role = "driver"
+  expand = true
+  disable = [
+    "PROPAG_WAM", "NEWWIND", "GSTATS", "FREQUENCY_FIELD%INIT",
+    "ENVIRONMENT_FIELD%INIT", "FORCING_FIELDS_FIELD%INIT",
+    "WAVE2OCEAN_FIELD%INIT", "INTGT_PARAM_FIELDS_FIELD%INIT",
+    "FREQUENCY_FIELD%UPDATE_DEVICE", "ENVIRONMENT_FIELD%UPDATE_DEVICE",
+    "FORCING_FIELDS_FIELD%UPDATE_DEVICE",
+    "WAVE2OCEAN_FIELD%UPDATE_DEVICE",
+    "INTGT_PARAM_FIELDS_FIELD%UPDATE_DEVICE",
+    "SOURCE_CONTRIBS_FIELD%INIT", "SOURCE_CONTRIBS_FIELD%UPDATE_DEVICE",
+    "INCDATE", "DR_HOOK", "omp_get_wtime", "abort1", "wam_user_clock",
+    "FREQUENCY_FIELD%ENSURE_HOST", "ENVIRONMENT_FIELD%ENSURE_HOST",
+    "FORCING_FIELDS_FIELD%ENSURE_HOST", "WAVE2OCEAN_FIELD%ENSURE_HOST",
+    "INTGT_PARAM_FIELDS_FIELD%ENSURE_HOST",
+    "SOURCE_CONTRIBS_FIELD%INIT", "SOURCE_CONTRIBS_FIELD%ENSURE_HOST",
+    "FREQUENCY_FIELD%FINAL", "ENVIRONMENT_FIELD%FINAL",
+    "FORCING_FIELDS_FIELD%FINAL", "WAVE2OCEAN_FIELD%FINAL",
+    "INTGT_PARAM_FIELDS_FIELD%FINAL", "SOURCE_CONTRIBS_FIELD%INIT",
+    "SOURCE_CONTRIBS_FIELD%FINAL"
+  ]
+  enable_imports = false
+  real_kind = 'JWRB'
 
-[[dimension]]
-name = "vertical"
-size = "NFRE"
-index = "M"
+# Define indices and bounds for array dimensions
+[dimensions]
 
-[[dimension]]
-name = "block_dim"
-size = "NCHNK"
-index = "ICHNK"
+[dimensions.horizontal]
+  size = "KIJL"
+  index = "IJ"
+  bounds = ["KIJS", "KIJL"]
+  aliases = ["NPROMA_WAM"]
+
+[dimensions.vertical]
+  size = "NFRE"
+  index = "M"
+
+[dimensions.block_dim]
+  size = "NCHNK"
+  index = "ICHNK"
diff --git a/src/ecwam/wam_init_gpu_mod.F90 b/src/ecwam/wam_init_gpu_mod.F90
@@ -0,0 +1,30 @@
+! (C) Copyright 1989- ECMWF.
+!
+! This software is licensed under the terms of the Apache Licence Version 2.0
+! which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+! In applying this licence, ECMWF does not waive the privileges and immunities
+! granted to it by virtue of its status as an intergovernmental organisation
+! nor does it submit to any jurisdiction.
+!
+
+       MODULE WAM_INIT_GPU_MOD
+       CONTAINS
+          SUBROUTINE WAM_INIT_GPU(IRANK)
+#ifdef _OPENACC
+             USE OPENACC
+#endif
+             USE PARKIND_WAVE, ONLY : JWIM
+             IMPLICIT NONE
+
+             INTEGER(KIND=JWIM), INTENT(IN) :: IRANK
+             INTEGER :: DEVTYPE, DEVNUM, DEV
+
+
+#ifdef _OPENACC
+             DEVTYPE = ACC_GET_DEVICE_TYPE()
+             DEVNUM = ACC_GET_NUM_DEVICES(DEVTYPE)
+             DEV = MOD(IRANK-1, DEVNUM)
+             CALL ACC_SET_DEVICE_NUM(DEV, DEVTYPE)
+#endif
+          END SUBROUTINE WAM_INIT_GPU
+       END MODULE WAM_INIT_GPU_MOD
diff --git a/src/ecwam/wvwaminit.F90 b/src/ecwam/wvwaminit.F90
@@ -33,9 +33,6 @@ SUBROUTINE WVWAMINIT (LLCOUPLED, IULOG, LLRNL,                    &
 
 ! ----------------------------------------------------------------------
 
-#if defined(WAM_PHYS_GPU) || defined(WAM_CUDA)
-      USE OPENACC
-#endif
       USE PARKIND_WAVE, ONLY : JWIM, JWRB, JWRU
 
       USE YOWMAP   , ONLY : AMOSOP   ,AMONOP   ,IQGAUSS
@@ -51,6 +48,7 @@ SUBROUTINE WVWAMINIT (LLCOUPLED, IULOG, LLRNL,                    &
 
       USE MPL_MODULE, ONLY : MPL_MYRANK, MPL_NPROC, MPL_BARRIER
       USE YOMHOOK  , ONLY : LHOOK,   DR_HOOK, JPHOOK
+      USE WAM_INIT_GPU_MOD, ONLY : WAM_INIT_GPU
 
 ! ----------------------------------------------------------------------
       IMPLICIT NONE
@@ -77,7 +75,6 @@ SUBROUTINE WVWAMINIT (LLCOUPLED, IULOG, LLRNL,                    &
       LOGICAL, INTENT(IN) :: LLCOUPLED
       LOGICAL :: LLEXIST
       LOGICAL, SAVE :: LFRST
-      INTEGER :: DEVTYPE, DEVNUM, DEV
 
       DATA LFRST /.TRUE./
 
@@ -91,10 +88,7 @@ SUBROUTINE WVWAMINIT (LLCOUPLED, IULOG, LLRNL,                    &
       NPROC = MPL_NPROC()
 
 #if defined(WAM_PHYS_GPU) || defined(WAM_CUDA)
-      DEVTYPE = ACC_GET_DEVICE_TYPE()
-      DEVNUM = ACC_GET_NUM_DEVICES(DEVTYPE)
-      DEV = MOD(IRANK-1, DEVNUM)
-      CALL ACC_SET_DEVICE_NUM(DEV, DEVTYPE)
+      CALL WAM_INIT_GPU(IRANK)
 #endif
 
 !     STANDARD OUTPUT UNIT