From b3d781c2b69f49605c23998c61e5604eec512791 Mon Sep 17 00:00:00 2001
From: "Benjamin.Blake EMC" <Benjamin.Blake@v72a3.ncep.noaa.gov>
Date: Thu, 3 Oct 2019 16:48:49 +0000
Subject: [PATCH] Changes to support OCONUS domains and chgres cube on WCOSS

---
 Externals.cfg                  |  2 +-
 rocoto/sites/wcoss_cray.ent    | 12 ++++++------
 rocoto/sites/wcoss_dell_p3.ent | 11 ++++++-----
 scripts/exregional_make_bc.sh  |  2 +-
 scripts/exregional_make_ic.sh  |  2 +-
 sorc/link_fix.sh               |  7 +------
 ush/run_commands_bc.sh         |  2 +-
 7 files changed, 17 insertions(+), 21 deletions(-)

diff --git a/Externals.cfg b/Externals.cfg
index e3477b386..21156b5d4 100644
--- a/Externals.cfg
+++ b/Externals.cfg
@@ -6,7 +6,7 @@ local_path = sorc/regional_utils.fd
 required = True
 
 [nemsfv3gfs]
-branch = develop
+branch = master
 protocol = git
 repo_url = gerrit:NEMSfv3gfs
 local_path = sorc/regional_forecast.fd
diff --git a/rocoto/sites/wcoss_cray.ent b/rocoto/sites/wcoss_cray.ent
index b3a95c4dd..15e3ac319 100644
--- a/rocoto/sites/wcoss_cray.ent
+++ b/rocoto/sites/wcoss_cray.ent
@@ -23,17 +23,17 @@
 
   <!ENTITY GET_INPUT_RESOURCES "<cores>1</cores><walltime>03:00:00</walltime>">
 
-  <!ENTITY MAKE_IC_RESOURCES "<nodes>1:ppn=1:tpp=1</nodes><envar><name>TOTAL_TASKS</name><value>1</value></envar><envar><name>NCTSK</name><value>1</value></envar><envar><name>OMP_THREADS</name><value>1</value></envar><walltime>00:15:00</walltime>">
+  <!ENTITY MAKE_IC_RESOURCES "<nodes>4:ppn=4:tpp=1</nodes><envar><name>TOTAL_TASKS</name><value>16</value></envar><envar><name>NCTSK</name><value>4</value></envar><envar><name>OMP_THREADS</name><value>1</value></envar><walltime>00:15:00</walltime>">
 
-  <!ENTITY MAKE_BC_RESOURCES "<nodes>20:ppn=1:tpp=1</nodes><envar><name>TOTAL_TASKS</name><value>20</value></envar><envar><name>NCTSK</name><value>1</value></envar><envar><name>OMP_THREADS</name><value>24</value></envar><walltime>00:30:00</walltime>">
+  <!ENTITY MAKE_BC_RESOURCES "<nodes>4:ppn=4:tpp=1</nodes><envar><name>TOTAL_TASKS</name><value>16</value></envar><envar><name>NCTSK</name><value>4</value></envar><envar><name>OMP_THREADS</name><value>1</value></envar><walltime>00:50:00</walltime>">
 
 <!-- some of these are domain specific -->
   <!ENTITY FORECAST_EXTRA "<envar><name>NCTSK</name><value>12</value></envar><envar><name>OMP_THREADS</name><value>2</value></envar><walltime>02:40:00</walltime>">
-  <!ENTITY FORECAST_TM00_RESOURCES_hi "<nodes>6:ppn=12:tpp=2</nodes><envar><name>TOTAL_TASKS</name><value>72</value></envar>&FORECAST_EXTRA;">
+  <!ENTITY FORECAST_TM00_RESOURCES_hi "<nodes>6:ppn=12:tpp=2</nodes><envar><name>TOTAL_TASKS</name><value>72</value></envar><envar><name>TASK_X</name><value>6</value></envar><envar><name>TASK_Y</name><value>10</value></envar><envar><name>WG</name><value>1</value></envar><envar><name>WTPG</name><value>12</value></envar>&FORECAST_EXTRA;">
   <!ENTITY FORECAST_TM00_RESOURCES_conus "<nodes>76:ppn=12:tpp=2</nodes><envar><name>TOTAL_TASKS</name><value>912</value></envar><envar><name>TASK_X</name><value>16</value></envar><envar><name>TASK_Y</name><value>48</value></envar><envar><name>WG</name><value>3</value></envar><envar><name>WTPG</name><value>48</value></envar>&FORECAST_EXTRA;">
-  <!ENTITY FORECAST_TM00_RESOURCES_ak "<nodes>60:ppn=12:tpp=2</nodes><envar><name>TOTAL_TASKS</name><value>720</value></envar>&FORECAST_EXTRA;">
-  <!ENTITY FORECAST_TM00_RESOURCES_pr "<nodes>9:ppn=12:tpp=2</nodes><envar><name>TOTAL_TASKS</name><value>108</value></envar>&FORECAST_EXTRA;">
-  <!ENTITY FORECAST_TM00_RESOURCES_guam "<nodes>6:ppn=12:tpp=2</nodes><envar><name>TOTAL_TASKS</name><value>72</value></envar>&FORECAST_EXTRA;">
+  <!ENTITY FORECAST_TM00_RESOURCES_ak "<nodes>60:ppn=12:tpp=2</nodes><envar><name>TOTAL_TASKS</name><value>720</value></envar><envar><name>TASK_X</name><value>14</value></envar><envar><name>TASK_Y</name><value>48</value></envar><envar><name>WG</name><value>2</value></envar><envar><name>WTPG</name><value>24</value></envar>&FORECAST_EXTRA;">
+  <!ENTITY FORECAST_TM00_RESOURCES_pr "<nodes>9:ppn=12:tpp=2</nodes><envar><name>TOTAL_TASKS</name><value>108</value></envar><envar><name>TASK_X</name><value>8</value></envar><envar><name>TASK_Y</name><value>12</value></envar><envar><name>WG</name><value>1</value></envar><envar><name>WTPG</name><value>12</value></envar>&FORECAST_EXTRA;">
+  <!ENTITY FORECAST_TM00_RESOURCES_guam "<nodes>6:ppn=12:tpp=2</nodes><envar><name>TOTAL_TASKS</name><value>72</value></envar> <envar><name>TASK_X</name><value>6</value></envar><envar><name>TASK_Y</name><value>10</value></envar><envar><name>WG</name><value>1</value></envar><envar><name>WTPG</name><value>12</value></envar>&FORECAST_EXTRA;">
 
   <!ENTITY POST_EXTRA "<envar><name>NCTSK</name><value>24</value></envar><envar><name>OMP_THREADS</name><value>1</value></envar><walltime>00:20:00</walltime>">
   <!ENTITY POST_RESOURCES_conus "<nodes>3:ppn=24</nodes><envar><name>TOTAL_TASKS</name><value>72</value></envar>&POST_EXTRA;">
diff --git a/rocoto/sites/wcoss_dell_p3.ent b/rocoto/sites/wcoss_dell_p3.ent
index fcb444729..86fd5e433 100644
--- a/rocoto/sites/wcoss_dell_p3.ent
+++ b/rocoto/sites/wcoss_dell_p3.ent
@@ -25,14 +25,15 @@
 
   <!ENTITY MAKE_IC_RESOURCES "<nodes>1:ppn=1:tpp=24</nodes><envar><name>TOTAL_TASKS</name><value>24</value></envar><envar><name>OMP_THREADS</name><value>1</value></envar><walltime>00:30:00</walltime><native>-R affinity[core]</native>">
 
-  <!ENTITY MAKE_BC_RESOURCES "<nodes>20:ppn=1</nodes><envar><name>TOTAL_TASKS</name><value>480</value></envar><envar><name>OMP_THREADS</name><value>1</value></envar><walltime>00:30:00</walltime><native>-R affinity[core]</native>">
+  <!ENTITY MAKE_BC_RESOURCES "<nodes>1:ppn=1:tpp=24</nodes><envar><name>TOTAL_TASKS</name><value>24</value></envar><envar><name>OMP_THREADS</name><value>1</value></envar><walltime>01:30:00</walltime><native>-R affinity[core]</native>">
 
   <!ENTITY FORECAST_EXTRA "<envar><name>NCTSK</name><value>12</value></envar><envar><name>OMP_THREADS</name><value>2</value></envar><walltime>01:30:00</walltime><native>-R affinity[core\(2\):distribute=balance]</native>">
   <!ENTITY FORECAST_TM00_RESOURCES_conus "<nodes>76:ppn=12:tpp=2</nodes><envar><name>TOTAL_TASKS</name><value>912</value></envar><envar><name>TASK_X</name><value>16</value></envar><envar><name>TASK_Y</name><value>48</value></envar><envar><name>WG</name><value>3</value></envar><envar><name>WTPG</name><value>48</value></envar>&FORECAST_EXTRA;">
-  <!ENTITY FORECAST_TM00_RESOURCES_ak "<nodes>60:ppn=12:tpp=2</nodes><envar><name>TOTAL_TASKS</name><value>720</value></envar>&FORECAST_EXTRA;">
-  <!ENTITY FORECAST_TM00_RESOURCES_hi "<nodes>6:ppn=12:tpp=2</nodes><envar><name>TOTAL_TASKS</name><value>72</value></envar>&FORECAST_EXTRA;">
-  <!ENTITY FORECAST_TM00_RESOURCES_pr "<nodes>9:ppn=12:tpp=2</nodes><envar><name>TOTAL_TASKS</name><value>108</value></envar>&FORECAST_EXTRA;">
-  <!ENTITY FORECAST_TM00_RESOURCES_guam "<nodes>6:ppn=12:tpp=2</nodes><envar><name>TOTAL_TASKS</name><value>72</value></envar>&FORECAST_EXTRA;">
+  <!ENTITY FORECAST_TM00_RESOURCES_ak "<nodes>60:ppn=12:tpp=2</nodes><envar><name>TOTAL_TASKS</name><value>720</value></envar><envar><name>TASK_X</name><value>14</value></envar><envar><name>TASK_Y</name><value>48</value></envar><envar><name>WG</name><value>2</value></envar><envar><name>WTPG</name><value>24</value></envar>&FORECAST_EXTRA;">
+  <!ENTITY FORECAST_TM00_RESOURCES_pr "<nodes>9:ppn=12:tpp=2</nodes><envar><name>TOTAL_TASKS</name><value>108</value></envar><envar><name>TASK_X</name><value>8</value></envar><envar><name>TASK_Y</name><value>12</value></envar><envar><name>WG</name><value>1</value></envar><envar><name>WTPG</name><value>12</value></envar>&FORECAST_EXTRA;">
+  <!ENTITY FORECAST_TM00_RESOURCES_guam "<nodes>6:ppn=12:tpp=2</nodes><envar><name>TOTAL_TASKS</name><value>72</value></envar> <envar><name>TASK_X</name><value>6</value></envar><envar><name>TASK_Y</name><value>10</value></envar><envar><name>WG</name><value>1</value></envar><envar><name>WTPG</name><value>12</value></envar>&FORECAST_EXTRA;">
+  <!ENTITY FORECAST_TM00_RESOURCES_hi "<nodes>6:ppn=12:tpp=2</nodes><envar><name>TOTAL_TASKS</name><value>72</value></envar><envar><name>TASK_X</name><value>6</value></envar><envar><name>TASK_Y</name><value>10</value></envar><envar><name>WG</name><value>1</value></envar><envar><name>WTPG</name><value>12</value></envar>&FORECAST_EXTRA;">
+
 
   <!ENTITY POST_EXTRA "<envar><name>NCTSK</name><value>14</value></envar><envar><name>OMP_THREADS</name><value>1</value></envar><walltime>00:20:00</walltime><native>-R affinity[core]</native>">
   <!ENTITY POST_RESOURCES_conus "<nodes>2:ppn=14</nodes><envar><name>TOTAL_TASKS</name><value>28</value></envar>&POST_EXTRA;">
diff --git a/scripts/exregional_make_bc.sh b/scripts/exregional_make_bc.sh
index 21783f0dc..e26ed5eff 100755
--- a/scripts/exregional_make_bc.sh
+++ b/scripts/exregional_make_bc.sh
@@ -87,7 +87,7 @@ cat <<EOF >fort.41
 /
 EOF
 
-  time ${APRUNC} -l ./regional_chgres_cube.x
+  time ${APRUNC} ./regional_chgres_cube.x
   hour=`expr $hour + $hour_inc`
 
 #
diff --git a/scripts/exregional_make_ic.sh b/scripts/exregional_make_ic.sh
index f86ba040f..48ca339c6 100755
--- a/scripts/exregional_make_ic.sh
+++ b/scripts/exregional_make_ic.sh
@@ -73,7 +73,7 @@ cat <<EOF >fort.41
 /
 EOF
 
-time ${APRUNC} -l ./regional_chgres_cube.x
+time ${APRUNC} ./regional_chgres_cube.x
 
 #
 # move output files to save directory
diff --git a/sorc/link_fix.sh b/sorc/link_fix.sh
index 4602e6580..af58f5c33 100755
--- a/sorc/link_fix.sh
+++ b/sorc/link_fix.sh
@@ -3,12 +3,7 @@ set -xeu
 
 source ./machine-setup.sh > /dev/null 2>&1
 
-LINK="ln -sf"
-if [[ $# -ge 1 ]]; then
-  if [[ $1 = "nco" ]]; then
-    LINK="cp -rp"
-  fi
-fi
+LINK="cp -rp"
 
 pwd=$(pwd -P)
 
diff --git a/ush/run_commands_bc.sh b/ush/run_commands_bc.sh
index 518cea81a..ec2d6a2ff 100644
--- a/ush/run_commands_bc.sh
+++ b/ush/run_commands_bc.sh
@@ -18,7 +18,7 @@ if [ "$machine" = wcoss_cray ]; then
 elif [ "$machine" = wcoss_dell_p3 ]; then
   export APRUNS=${APRUNS:-"time"}
   export APRUNF=${APRUNF:-"mpirun cfp"}
-  export APRUNC=${APRUNC:-"time"}
+  export APRUNC=${APRUNC:-"mpirun"}
   export APRUNO="time"
   export BACKGROUND=""
 elif [ "$machine" = hera ]; then