New pyft version (parallel tool)

UMR-CNRM · Oct 3, 2024 · 2ba3ba6 · 2ba3ba6
1 parent fef003b
commit 2ba3ba6
Show file tree

Hide file tree

Showing 4 changed files with 78 additions and 67 deletions.
diff --git a/build/with_fcm/arch/arch-pgi_CPU_preGPU_MFflavour.env b/build/with_fcm/arch/arch-pgi_CPU_preGPU_MFflavour.env
@@ -22,25 +22,28 @@ stackMode=AROME
 #* do not modify turb_hor* or mode_rotate_wind.F90 (non column subroutines)
 #* remove call to condensation from lima, do not touch lima routines (due to implicit dimensions at declaration)
 #* remove problematic call to BL_DEPTH_DIAG_3D from the unused (in testprogs and AROME) BL_DEPTH_DIAG_1D
+#* do not touch .f files which will be included
 #  this part is written differently for each model. For MesoNH, LES diag needs this call
-specificTransfos="^progs/:=:
-^rain_ice/:=:
-^rain_ice_old/:=:
-^ice_adjust/:=:
-^turb_mnh/:=:
-^shallow/:=:
-^support/:=:
-^ext/:=:
-^micro/minpack.F90:=:
-^aux/tools.F90:=:
-^turb/mode_turb_hor:=:--empty --simplify
-^turb/mode_rotate_wind.F90:=:--empty --simplify
-^turb/mode_tridiag_w.F90:=:--empty --simplify
-^micro/mode_lima:=:
-^micro/lima:=:
-^micro/lima_adjust_split.F90:=:--removeCall sub:LIMA_ADJUST_SPLIT CONDENSATION"
+specificTransfos="^(./)?progs/:=:
+^(./)?rain_ice/:=:
+^(./)?rain_ice_old/:=:
+^(./)?ice_adjust/:=:
+^(./)?turb_mnh/:=:
+^(./)?shallow/:=:
+^(./)?support/:=:
+^(./)?ext/:=:
+^(./)?micro/minpack\.F90:=:
+^(./)?aux/tools\.F90:=:
+^(./)?turb/mode_turb_hor:=:--empty --simplify
+^(./)?turb/mode_rotate_wind\.F90:=:--empty --simplify
+^(./)?turb/mode_tridiag_w\.F90:=:--empty --simplify
+^(./)?micro/mode_lima:=:
+^(./)?micro/lima:=:
+^(./)?micro/lima_adjust_split\.F90:=:--removeCall sub:LIMA_ADJUST_SPLIT CONDENSATION
+^(./)?micro/[_.a-z0-9]*\.h:=:
+^(./)?turb/[_.a-z0-9]*\.h:=:"
 
-#Temporary file for the description tree
+#Temporary file for the description tree, only for MesoNH and IAL
 descTree=${TMPDIR:-/tmp}/descTree_$$
 trap "\rm -f $descTree" EXIT
 
@@ -52,15 +55,15 @@ trap "\rm -f $descTree" EXIT
 PYFT_OPTS_MNH="--tree . --addIncludes --descTree $descTree --simplify --enableCache" #General options
 PYFT_OPTS_MNH="$PYFT_OPTS_MNH --inlineContainedSubroutinesPHYEX" #Inlining
 PYFT_OPTS_MNH="$PYFT_OPTS_MNH --deleteDrHook --deleteBudgetDDH" #Not available on GPU
-PYFT_OPTS_MNH="$PYFT_OPTS_MNH --addACC_routine_seq" #add 'acc routine' directive
+PYFT_OPTS_MNH="$PYFT_OPTS_MNH --addACCRoutineSeq" #add 'acc routine' directive
 PYFT_OPTS_MNH="$PYFT_OPTS_MNH --stopScopes sub:SHALLOW_MF#sub:ICE_ADJUST#sub:RAIN_ICE#sub:RAIN_ICE_OLD#sub:TURB#sub:LIMA_ADJUST_SPLIT#sub:LIMA"
 PYFT_OPTS_MNH="$PYFT_OPTS_MNH --setFalseIfStmt TURBN%LTURB_DIAG --setFalseIfStmt TURBN%LTURB_FLX" #remove diagnostic output
 PYFT_OPTS_MNH="$PYFT_OPTS_MNH --buildACCTypeHelpers"
 PYFT_OPTS_MNH="$PYFT_OPTS_MNH --addArrayParentheses --expandAllArraysPHYEX --attachArraySpecToEntity --applyCPPifdef %REPRO48 --replaceAutomaticWithAllocatable"
 PYFT_OPTS_MNH="$PYFT_OPTS_MNH
 $specificTransfos
-^turb/mode_bl_depth_diag.F90:=:--removeCall module:MODE_BL_DEPTH_DIAG/sub:BL_DEPTH_DIAG_1D BL_DEPTH_DIAG_3D $PYFT_OPTS_MNH
-^micro/rain_ice.F90:=:--removeCall sub:RAIN_ICE ELEC_TENDENCIES $PYFT_OPTS_MNH"
+^(./)?turb/mode_bl_depth_diag.F90:=:--removeCall module:MODE_BL_DEPTH_DIAG/sub:BL_DEPTH_DIAG_1D BL_DEPTH_DIAG_3D $PYFT_OPTS_MNH
+^(./)?micro/rain_ice.F90:=:--removeCall sub:RAIN_ICE ELEC_TENDENCIES $PYFT_OPTS_MNH"
 
 #For IAL, stack transformation is replaced by allocatables
 #source this file, then: "export PYFT_OPTS_IAL; check_commit_ial.sh --prep_code-opts '--pyft_opts_env PYFT_OPTS_IAL' ...."
@@ -70,34 +73,33 @@ $specificTransfos
 PYFT_OPTS_IAL="--tree . --addIncludes --descTree $descTree --simplify --enableCache" #General options
 PYFT_OPTS_IAL="$PYFT_OPTS_IAL --inlineContainedSubroutinesPHYEX" #Inlining
 PYFT_OPTS_IAL="$PYFT_OPTS_IAL --deleteDrHook --deleteBudgetDDH" #Not available on GPU
-PYFT_OPTS_IAL="$PYFT_OPTS_IAL --addACC_routine_seq" #add 'acc routine' directive
+PYFT_OPTS_IAL="$PYFT_OPTS_IAL --addACCRoutineSeq" #add 'acc routine' directive
 PYFT_OPTS_IAL="$PYFT_OPTS_IAL --stopScopes sub:SHALLOW_MF#sub:ICE_ADJUST#sub:RAIN_ICE#sub:RAIN_ICE_OLD#sub:TURB#sub:LIMA_ADJUST_SPLIT#sub:LIMA"
 PYFT_OPTS_IAL="$PYFT_OPTS_IAL --setFalseIfStmt TURBN%LTURB_DIAG --setFalseIfStmt TURBN%LTURB_FLX" #remove diagnostic output
 PYFT_OPTS_IAL="$PYFT_OPTS_IAL --buildACCTypeHelpers"
 PYFT_OPTS_IAL="$PYFT_OPTS_IAL --addArrayParentheses --expandAllArraysPHYEX --attachArraySpecToEntity --applyCPPifdef REPRO48 --deleteNonColumnCallsPHYEX --replaceAutomaticWithAllocatable"
 PYFT_OPTS_IAL="$PYFT_OPTS_IAL
 $specificTransfos
-^turb/mode_bl_depth_diag.F90:=:--removeCall module:MODE_BL_DEPTH_DIAG/sub:BL_DEPTH_DIAG_1D BL_DEPTH_DIAG_3D $PYFT_OPTS_IAL
-^micro/rain_ice.F90:=:--removeCall sub:RAIN_ICE ELEC_TENDENCIES $PYFT_OPTS_IAL"
+^(./)?turb/mode_bl_depth_diag.F90:=:--removeCall module:MODE_BL_DEPTH_DIAG/sub:BL_DEPTH_DIAG_1D BL_DEPTH_DIAG_3D $PYFT_OPTS_IAL
+^(./)?micro/rain_ice.F90:=:--removeCall sub:RAIN_ICE ELEC_TENDENCIES $PYFT_OPTS_IAL"
 
 #For testprogs
 #check_commit_testprogs.sh -a <this conf>
 #For --removeIJDim, the USE_COLCALL compilation key must be set in the *.fcm file
 #--removeIJDim internally applies --addArrayParentheses --expandAllArraysPHYEX and, if --simplify is set, attachArraySpecToEntity
 #The --wrapH is automatically set by prep_code
 #The --mnhExpand is controlled by the different check_commit_* scripts
-#--addIncludes must be before --descTree because --addIncludes removes a wrong "file" node
-PYFT_OPTS="--tree . --addIncludes --descTree $descTree --simplify --enableCache" #General options
+#make_fcm automatically adds the --tree and --descTree options
+PYFT_OPTS="--addIncludes --simplify --enableCache" #General options
 PYFT_OPTS="$PYFT_OPTS --inlineContainedSubroutinesPHYEX" #Inlining
 PYFT_OPTS="$PYFT_OPTS --deleteDrHook --deleteBudgetDDH" #Not available on GPU
-#PYFT_OPTS="$PYFT_OPTS --addACC_routine_seq" #add 'acc routine' directive
 PYFT_OPTS="$PYFT_OPTS --stopScopes sub:SHALLOW_MF#sub:ICE_ADJUST#sub:RAIN_ICE#sub:RAIN_ICE_OLD#sub:TURB#sub:LIMA_ADJUST_SPLIT#sub:LIMA"
 PYFT_OPTS="$PYFT_OPTS --setFalseIfStmt TURBN%LTURB_DIAG --setFalseIfStmt TURBN%LTURB_FLX" #remove diagnostic output
 PYFT_OPTS="$PYFT_OPTS --buildACCTypeHelpers"
 PYFT_OPTS="$PYFT_OPTS --applyCPPifdef REPRO48 --deleteNonColumnCallsPHYEX --removeIJDim --addStack $stackMode"
-PYFT_OPTS="$PYFT_OPTS --removeACC --addACC_routine_seq" #remove pre-existing ACC directives, then add 'acc routine seq' directive
+PYFT_OPTS="$PYFT_OPTS --removeACC --addACCRoutineSeq" #remove pre-existing ACC directives, then add 'acc routine seq' directive
 PYFT_OPTS="$PYFT_OPTS
 $specificTransfos
-^turb/mode_bl_depth_diag.F90:=:--removeCall module:MODE_BL_DEPTH_DIAG/sub:BL_DEPTH_DIAG_1D BL_DEPTH_DIAG_3D $PYFT_OPTS
-^micro/rain_ice.F90:=:--removeCall sub:RAIN_ICE ELEC_TENDENCIES $PYFT_OPTS"
+^(./)?turb/mode_bl_depth_diag\.F90:=:--removeCall module:MODE_BL_DEPTH_DIAG/sub:BL_DEPTH_DIAG_1D BL_DEPTH_DIAG_3D $PYFT_OPTS
+^(./)?micro/rain_ice\.F90:=:--removeCall sub:RAIN_ICE ELEC_TENDENCIES $PYFT_OPTS"
 
diff --git a/build/with_fcm/make_fcm.sh b/build/with_fcm/make_fcm.sh
@@ -392,6 +392,7 @@ if [ $packupdate -eq 1 -o $packcreation -eq 1 ]; then
     rm -rf src
   fi
   . arch.env
+  [ -z ${PYFT_OPTS+x} ] && PYFT_OPTS=''
   export PYFT_OPTS
 
   # Populate the source directory with (modified) PHYEX source code
@@ -411,20 +412,28 @@ if [ $packupdate -eq 1 -o $packcreation -eq 1 ]; then
   fi
   PHYEXTOOLSDIR="$DIR/../../../tools" #if run from within a PHYEX repository
   UPDATEDPATH=$PATH
+
+  #Temporary file for the description tree
+  descTree=${TMPDIR:-/tmp}/descTree_$$
+  trap "\rm -f $descTree" EXIT
+
   which prep_code.sh > /dev/null || export UPDATEDPATH=$PHYEXTOOLSDIR:$PATH
   subs="$subs -s turb -s shallow -s turb_mnh -s micro -s aux -s ice_adjust -s rain_ice -s rain_ice_old -s support -s progs"
   if [ "$fromdir" == '' ]; then
     echo "Clone repository, and checkout commit $commit (using prep_code.sh)"
     if [[ $commit == testprogs${separator}* || $commit == offline${separator}* ]]; then
-      PATH=$UPDATEDPATH prep_code.sh --pyft_opts_env PYFT_OPTS -c $commit src #This commit is ready for inclusion
+      #This commit is ready for inclusion
+      PATH=$UPDATEDPATH prep_code.sh -c $commit src
     else
-      PATH=$UPDATEDPATH prep_code.sh --pyft_opts_env PYFT_OPTS -c $commit $expand_options $subs -m offline src
+      PATH=$UPDATEDPATH prep_code.sh --pyft_opts_env PYFT_OPTS -c $commit $expand_options $subs \
+                                     -m offline src --useParallelPyft -- --tree . --descTree $descTree
     fi
   else
     echo "Copy $fromdir"
     mkdir src
     scp -q -r $fromdir/src src/
-    PATH=$UPDATEDPATH prep_code.sh --pyft_opts_env PYFT_OPTS $expand_options $subs -m offline src
+    PATH=$UPDATEDPATH prep_code.sh --pyft_opts_env PYFT_OPTS $expand_options $subs \
+                                   -m offline src --useParallelPyft -- --tree . --descTree $descTree
   fi
 
   # Add some code
@@ -445,8 +454,8 @@ if [ $packupdate -eq 1 -o $packcreation -eq 1 ]; then
     pybinding.py micro/lima.F90 sub:LIMA pyphyex.F90 ../build/bin/pyphyex.py ./libphyex.so
   else
     cat <<....EOF > pyphyex.F90
-    SUBROUTINE PYPHYEX
-    END SUBROUTINE PYPHYEX
+    SUBROUTINE PYPHYEXSUB
+    END SUBROUTINE PYPHYEXSUB
 ....EOF
   fi
   ln -s ../../fiat/src fiat

diff --git a/tools/INSTALL.sh b/tools/INSTALL.sh
@@ -4,7 +4,7 @@
 set -e
 set -o pipefail #abort if left command on a pipe fails
 
-pyft_version=f63de396176e3e91a5cdedae2d098dbbe3497ae2
+pyft_version=76f61bcd9eae523ced16e8d854e0f065fa82078e
 
 #This script installs PHYEX
 #Call the script with the -h option to get more information.

diff --git a/tools/prep_code.sh b/tools/prep_code.sh
@@ -33,6 +33,7 @@ function usage {
   echo "-v                    add verbosity (up to 3 -v)"
   echo "--pyft_opts_env VAR   name of an environment variable containing options to use to call"
   echo "                      the pyft_tool.py script"
+  echo "--useParallelPyft     use the parallel version of the pyft tool (pyft_parallel_tool.py)"
   echo "-- PYFT_OPTIONS       everything after '--' are used as options for pyft_tool.py"
   echo "                      These options are used for all the files."
   echo ""
@@ -81,6 +82,7 @@ subs=""
 renameFf=0
 verbose=0
 ilooprm=0
+useParallelPyft=0
 forpyft=0
 pyft_opts_env=""
 
@@ -109,6 +111,7 @@ while [ -n "$1" ]; do
     '--ilooprm') ilooprm=1;;
     '--repo') repository=$2; shift;;
     '-v') verbose=$(($verbose+1));;
+    '--useParallelPyft') useParallelPyft=1;;
     '--pyft_opts_env') pyft_opts_env=$2; shift;;
     '--') forpyft=1;;
      *) if [ $forpyft -eq 0 ]; then
@@ -120,11 +123,6 @@ while [ -n "$1" ]; do
   shift
 done
 
-if [ "$pyft_opts_env" != "" ]; then
-  #pyft_opts_env contains the name of the environment variable to use
-  pyft_opts_env=${!pyft_opts_env}
-  #now, pyft_opts_env contains the configuration to use
-fi
 if [ $verbose -ge 3 ]; then
   set -x
 fi
@@ -297,7 +295,11 @@ if [ "$pyft_opts_env" != "" -o -n "${pyft_options-}" ]; then
   [ $verbose -gt 0 ] && echo "Applying pyft_tool"
 
   #Update PATH and PYTHONPATH if needed
-  which pyft_tool.py > /dev/null || . $PHYEXTOOLSDIR/site/pyft/bin/env.sh
+  if [ $useParallelPyft -eq 1 ]; then
+    which pyft_parallel_tool.py > /dev/null || . $PHYEXTOOLSDIR/site/pyft/bin/env.sh
+  else
+    which pyft_tool.py > /dev/null || . $PHYEXTOOLSDIR/site/pyft/bin/env.sh
+  fi
 
   if [ -n "${model-}" ]; then
     reps=$subs
@@ -307,35 +309,33 @@ if [ "$pyft_opts_env" != "" -o -n "${pyft_options-}" ]; then
       reps="$reps src/*/$sub"
     done
   fi
-  for rep in $reps; do
-    if [ -d $rep ]; then
-      find $rep -type f -not -name '.*.swp'  -not -name '.*.swo' | while read file; do
-        if [ "$(echo $file | grep '\.')" != '' -a $(echo $file | rev | cut -d. -f1 | rev) != 'fypp' ]; then
-          #Files without extension are certainly not source code files
-          #.fypp files cannot be read by pyft_tool.py
-          extra_opts=""
-          if [ "$pyft_opts_env" != "" ]; then
-            while read line; do
-              if echo $line | grep ':=:' > /dev/null; then
-                #This line has the form FILE_DESCRIPTOR:OPTIONS
-                fd="$(echo $line | awk -F :=: '{print $1}')"
-                if echo $file | grep -e "$fd" > /dev/null; then
-                  extra_opts=$(echo $line | awk -F :=: '{$1=""; print $0}') #equivalent to cut -d:=: -f2- (multi characters separator are forbiden with cut)
-                fi
-              else
-                extra_opts=$line
-              fi
-            done < <(echo "$pyft_opts_env")
-          fi
-          if [ "$extra_opts" != "" -o -n "${pyft_options-}" ]; then
-            cmd="pyft_tool.py --wrapH $pyft_options $extra_opts" #--wrapH allows to deal with h files
-            [ $verbose -gt 1 ] && echo $cmd "$file"
-            $cmd "$file"
-          fi
+
+  extra_opts=""
+  if [ "$pyft_opts_env" != "" ]; then
+    extra_opts="--optsByEnv $pyft_opts_env"
+  fi
+
+  if [ "$extra_opts" != "" -o -n "${pyft_options-}" ]; then
+    if [ $useParallelPyft -eq 1 ]; then
+      cmd="pyft_parallel_tool.py --wrapH $pyft_options $extra_opts --nbPar 8" #--wrapH allows to deal with h files
+      [ $verbose -gt 1 ] && echo $cmd
+      $cmd
+    else
+      for rep in $reps; do
+        if [ -d $rep ]; then
+          find $rep -type f -not -name '.*.swp'  -not -name '.*.swo' | while read file; do
+            if [ "$(echo $file | grep '\.')" != '' -a $(echo $file | rev | cut -d. -f1 | rev) != 'fypp' ]; then
+              #Files without extension are certainly not source code files
+              #.fypp files cannot be read by pyft_tool.py
+              cmd="pyft_tool.py --wrapH $pyft_options $extra_opts" #--wrapH allows to deal with h files
+              [ $verbose -gt 1 ] && echo $cmd "$file"
+              $cmd "$file"
+            fi
+          done
         fi
       done
     fi
-  done
+  fi
 fi
 
 ###### PUSH