Merge pull request #167 from bertinia/master

major upgrades
NCAR · Sep 10, 2018 · 6b3b9fc · 6b3b9fc
2 parents 19c1e01 + 20227fc
commit 6b3b9fc
Show file tree

Hide file tree

Showing 49 changed files with 1,859 additions and 1,207 deletions.
diff --git a/Config/config_postprocess.xml b/Config/config_postprocess.xml
@@ -43,12 +43,12 @@
 	     desc="post processing directory location on local machine where cesm-env2 python virtualenv is located."
 	     ></entry> 
 
-      <entry id="POSTPROCESS_PATH_GEYSER"
+      <entry id="POSTPROCESS_PATH_DAV"
 	     type="string"
 	     valid_values=""
 	     value="inherit"  
 	     group="case_info"
-	     desc="post processing directory location on local machine where geyser specific cesm-env2 python virtualenv is located."
+	     desc="post processing directory location on local machine where DAV specific cesm-env2 python virtualenv is located."
 	     ></entry> 
 
       <entry id="DOUT_S_ROOT"

diff --git a/Machines/cheyenne_modules b/Machines/cheyenne_modules
@@ -1,29 +1,21 @@
-#!/bin/sh -x
+#!/bin/bash -l
 
 echo "Python boot-strap modules for machine cheyenne"
-. /glade/u/apps/ch/opt/lmod/7.2.1/lmod/lmod/init/bash
 
-## Order is important!
-##module restore system
-module purge
-module load ncarenv/1.2
+module load python/2.7.14
 module load intel/17.0.1
-module load ncarcompilers/0.4.1
+module load ncarenv
+module load ncarcompilers
 module load mpt/2.15f
-module load python/2.7.13
-module load numpy/1.12.0
-module load scipy/0.18.1
-module load mpi4py/2.0.0-mpt
-module load netcdf/4.4.1.1
-module load pynio/1.4.1
-module load matplotlib/2.0.0
-module load nco/4.6.2
+module load netcdf/4.6.1
+module load nco/4.7.4
 module load ncl/6.4.0
-module load netcdf4-python/1.2.7
-module load cf_units/1.1.3
-module load pyngl/1.5.0b
 
-# prepend the virtualenv into the PATH
-PATH=/glade/u/apps/ch/opt/python/2.7.13/gnu/6.2.0/bin:${PATH}
-export PATH
+# clone the ncat virtualenv first with helper script ncar_pylib
+# use "ncar_pylib --help" to see all options
+ncar_pylib -c 20180705 ${pp_dir}/cesm-env2
+
+export PYTHONPATH=${pp_dir}/cesm-env2/lib/python2.7/site-packages
+
+module list
 
diff --git a/Machines/dav_modules b/Machines/dav_modules
@@ -0,0 +1,21 @@
+#!/bin/bash -l
+
+echo "Python boot-strap modules for NCAR DAV"
+
+module load python/2.7.14
+module load intel/17.0.1
+module load ncarenv
+module load ncarcompilers
+module load impi
+module load netcdf/4.6.1
+module load nco/4.7.4
+module load ncl/6.4.0
+
+# clone the ncat virtualenv first with helper script ncar_pylib
+# use "ncar_pylib --help" to see all options
+ncar_pylib -c 20180510 ${pp_dir}/cesm-env2
+
+export PYTHONPATH=${pp_dir}/cesm-env2/lib/python2.7/site-packages
+
+module list
+
diff --git a/Machines/geyser_modules b/Machines/geyser_modules
@@ -1,32 +1,21 @@
-#!/bin/sh
+#!/bin/bash -l
 
 echo "Python boot-strap modules for machine geyser"
-. /glade/apps/opt/lmod/lmod/init/bash
 
-module load python/2.7.7
-module load intel/16.0.3
+module load python/2.7.14
+module load intel/17.0.1
 module load ncarenv
-module load ncarbinlibs
 module load ncarcompilers
-module load slurm/17
-module load impi/5.1.1.109
-module load numpy/1.11.0
-module load scipy/0.18.1
-module load mpi4py/2.0.0-impi
-module load pynio/1.4.1
-module load pyside/1.1.2
-module load matplotlib/1.5.1
-module load netcdf/4.3.0
-module load nco/4.4.4
+module load impi
+module load netcdf/4.6.1
+module load nco/4.7.4
 module load ncl/6.4.0
-module load netcdf4python/1.2.4
-module load pyngl/1.4.0
 
-# prepend the virtualenv into the PATH
-PATH=/glade/apps/contrib/virtualenv/12.0.7:${PATH}
-export PATH
+# clone the geyser virtualenv first with helper script ncar_pylib
+# use "ncar_pylib --help" to see all options
+ncar_pylib -c 20180510 ${pp_dir}/cesm-env2
 
-# may need basemap for ocn ecosys
+export PYTHONPATH=${pp_dir}/cesm-env2/lib/python2.7/site-packages
 
 module list
 
diff --git a/Machines/machine_postprocess.xml b/Machines/machine_postprocess.xml
@@ -2,40 +2,91 @@
 
 <machine_postprocess>
 
+  <machine name="dav" hostname="dav,geyser,caldera,pronghorn,casper">
+    <timeseries_pes  nodes="4" pes_per_node="16" wallclock="04:00:00" memory="100G">64</timeseries_pes>
+    <xconform_pes  nodes="4" pes_per_node="16" wallclock="06:00:00" memory="100G">64</xconform_pes>
+    <mpi_command>srun</mpi_command>
+    <pythonpath></pythonpath>
+    <f2py fcompiler="intelem" f77exec="/glade/apps/opt/modulefiles/ys/cmpwrappers/ifort">f2py</f2py>
+    <za>
+      <compiler>ifort</compiler>
+      <flags>-c -g -O2</flags>
+      <include>-I/glade/u/apps/dav/opt/netcdf/4.6.1/intel/17.0.1/include</include>
+      <libs>-L/glade/u/apps/dav/opt/netcdf/4.6.1/intel/17.0.1/lib -lnetcdff -lnetcdf</libs>
+    </za>
+    <reset_modules>
+      <module>module purge</module>
+    </reset_modules>
+    <modules>
+      <module>module load python/2.7.14</module>
+      <module>module load intel/17.0.1</module>
+      <module>module load ncarenv</module>
+      <module>module load ncarcompilers</module>
+      <module>module load impi</module>
+      <module>module load netcdf/4.6.1</module>
+      <module>module load nco/4.7.4</module>
+      <module>module load ncl/6.4.0</module>
+    </modules>
+    <components>
+      <component name="atm">
+	<averages_pes  nodes="4" pes_per_node="8" wallclock="02:00:00" memory="100G">32</averages_pes>
+	<diagnostics_pes  nodes="1" pes_per_node="16" wallclock="02:00:00" memory="100G">16</diagnostics_pes>
+	<regrid_pes  nodes="1" pes_per_node="6" wallclock="02:00:00" memory="100G">6</regrid_pes>
+	<obs_root>/glade/p/cesm/amwg/amwg_data</obs_root>
+      </component>
+      <component name="ice">
+	<averages_pes  nodes="4" pes_per_node="8" wallclock="02:00:00" memory="100G">32</averages_pes>
+	<diagnostics_pes  nodes="1" pes_per_node="4" wallclock="01:00:00" memory="100G">4</diagnostics_pes>
+	<obs_root>/glade/p/cesm/pcwg/ice/data</obs_root>
+      </component>
+      <component name="lnd">
+	<averages_pes  nodes="4" pes_per_node="8" wallclock="02:00:00" memory="100G">32</averages_pes>
+ 	<diagnostics_pes  nodes="1" pes_per_node="12" wallclock="02:00:00" memory="100G">12</diagnostics_pes>
+	<regrid_pes  nodes="1" pes_per_node="6" wallclock="02:00:00" memory="100G">6</regrid_pes>
+	<obs_root>/glade/p/cesm/lmwg/diag/lnd_diag_data</obs_root>
+      </component>
+      <component name="ocn">
+	<averages_pes  nodes="4" pes_per_node="8" wallclock="02:00:00" memory="100G">32</averages_pes>
+	<diagnostics_pes  nodes="1" pes_per_node="16" wallclock="02:00:00" memory="100G">16</diagnostics_pes>
+	<obs_root>/glade/p/cesm/</obs_root>
+      </component>
+      <component name="ilamb">
+	<diagnostics_pes  nodes="1" pes_per_node="2" wallclock="01:00:00" memory="100G">2</diagnostics_pes>
+	<initialize_pes  nodes="1" pes_per_node="1" wallclock="00:00:10" memory="1G">1</initialize_pes>
+	<obs_root>/glade/p/cesm/lmwg_dev/oleson/ILAMB/ILAMB_all</obs_root>
+      </component>
+      <component name="iomb">
+	<diagnostics_pes  nodes="1" pes_per_node="2" wallclock="01:00:00" memory="100G">2</diagnostics_pes>
+	<initialize_pes  nodes="1" pes_per_node="1" wallclock="00:00:10" memory="1G">1</initialize_pes>
+	<obs_root>/glade/p/cesm/omwg/obs_data/IOMB</obs_root>
+      </component>
+    </components>
+  </machine>
+
   <machine name="geyser" hostname="geyser">
     <timeseries_pes queue="geyser" nodes="4" pes_per_node="16" wallclock="04:00:00" memory="100G">64</timeseries_pes>
     <xconform_pes queue="geyser" nodes="4" pes_per_node="16" wallclock="06:00:00" memory="100G">64</xconform_pes>
     <mpi_command>srun</mpi_command>
-    <pythonpath>/glade/apps/opt/python/2.7.7/gnu-westmere/4.8.2/lib/python2.7/site-packages</pythonpath>
+    <pythonpath></pythonpath>
     <f2py fcompiler="intelem" f77exec="/glade/apps/opt/modulefiles/ys/cmpwrappers/ifort">f2py</f2py>
     <za>
       <compiler>ifort</compiler>
       <flags>-c -g -O2</flags>
-      <include>-I/glade/apps/opt/netcdf/4.3.3.1/intel/16.0.0/include</include>
-      <libs>-L/glade/apps/opt/netcdf/4.3.3.1/intel/16.0.0/lib -lnetcdff -lnetcdf</libs>
+      <include>-I/glade/u/apps/dav/opt/netcdf/4.6.1/intel/17.0.1/include</include>
+      <libs>-L/glade/u/apps/dav/opt/netcdf/4.6.1/intel/17.0.1/lib -lnetcdff -lnetcdf</libs>
     </za>
     <reset_modules>
       <module>module purge</module>
-      <module>module load python/2.7.7</module>
     </reset_modules>
     <modules>
-      <module>module load intel/16.0.3</module>
+      <module>module load python/2.7.14</module>
+      <module>module load intel/17.0.1</module>
       <module>module load ncarenv</module>
-      <module>module load ncarbinlibs</module>
       <module>module load ncarcompilers</module>
-      <module>module load slurm/17</module>
-      <module>module load impi/5.1.1.109</module>
-      <module>module load numpy/1.11.0</module>
-      <module>module load scipy/0.18.1</module>
-      <module>module load mpi4py/2.0.0-impi</module>
-      <module>module load pynio/1.4.1</module>
-      <module>module load pyside/1.1.2</module>
-      <module>module load matplotlib/1.5.1</module>
-      <module>module load netcdf/4.3.0</module>
-      <module>module load nco/4.4.4</module>
-      <module>module load netcdf4python/1.2.4</module>
+      <module>module load impi</module>
+      <module>module load netcdf/4.6.1</module>
+      <module>module load nco/4.7.4</module>
       <module>module load ncl/6.4.0</module>
-      <module>module load pyngl/1.4.0</module>
     </modules>
     <components>
       <component name="atm">
@@ -82,29 +133,21 @@
     <za>
       <compiler>ifort</compiler>
       <flags>-c -g -O2</flags>
-      <include>-I/glade/u/apps/ch/opt/netcdf/4.4.1.1/intel/16.0.3/include</include>
-      <libs>-L/glade/u/apps/ch/opt/netcdf/4.4.1.1/intel/16.0.3/lib -lnetcdff -lnetcdf</libs>
+      <include>-I/glade/u/apps/ch/opt/netcdf/4.6.1/intel/17.0.1/include</include>
+      <libs>-L/glade/u/apps/ch/opt/netcdf/4.6.1/intel/17.0.1/lib -lnetcdff -lnetcdf</libs>
     </za>
     <reset_modules>
       <module>module purge</module>
-      <module>module load ncarenv/1.2</module>
-      <module>module load intel/17.0.1</module>
-      <module>module load ncarcompilers/0.4.1</module>
-      <module>module load mpt/2.15f</module>
-      <module>module load python/2.7.13</module>
     </reset_modules>
     <modules>
-      <module>module load numpy/1.12.0</module>
-      <module>module load scipy/0.18.1</module>
-      <module>module load mpi4py/2.0.0-mpt</module>
-      <module>module load pynio/1.4.1</module>
-      <module>module load matplotlib/2.0.0</module>
-      <module>module load netcdf/4.4.1.1</module>
-      <module>module load nco/4.6.2</module>
-      <module>module load netcdf4-python/1.2.7</module>
-      <module>module load cf_units/1.1.3</module>
+      <module>module load python/2.7.14</module>
+      <module>module load intel/17.0.1</module>
+      <module>module load ncarenv</module>
+      <module>module load ncarcompilers</module>
+      <module>module load mpt/2.15f</module>
+      <module>module load netcdf/4.6.1</module>
+      <module>module load nco/4.7.4</module>
       <module>module load ncl/6.4.0</module>
-      <module>module load pyngl/1.5.0b</module>
     </modules>
     <components>
       <component name="atm">

diff --git a/Machines/machine_postprocess.xsd b/Machines/machine_postprocess.xsd
@@ -28,7 +28,7 @@
     <xs:complexType>
       <xs:simpleContent>
 	<xs:extension base="xs:string">
-	  <xs:attribute name="queue" use="required"/>
+	  <xs:attribute name="queue" use="optional"/>
 	  <xs:attribute name="nodes" use="optional"/>
 	  <xs:attribute name="pes_per_node" use="optional"/>
 	  <xs:attribute name="wallclock" use="required"/>
@@ -42,7 +42,7 @@
     <xs:complexType>
       <xs:simpleContent>
 	<xs:extension base="xs:string">
-	  <xs:attribute name="queue" use="required"/>
+	  <xs:attribute name="queue" use="optional"/>
 	  <xs:attribute name="nodes" use="optional"/>
 	  <xs:attribute name="pes_per_node" use="optional"/>
 	  <xs:attribute name="wallclock" use="required"/>
@@ -56,7 +56,7 @@
     <xs:complexType>
       <xs:simpleContent>
 	<xs:extension base="xs:string">
-	  <xs:attribute name="queue" use="required"/>
+	  <xs:attribute name="queue" use="optional"/>
 	  <xs:attribute name="nodes" use="optional"/>
 	  <xs:attribute name="pes_per_node" use="optional"/>
 	  <xs:attribute name="wallclock" use="required"/>
@@ -70,7 +70,7 @@
     <xs:complexType>
       <xs:simpleContent>
 	<xs:extension base="xs:string">
-	  <xs:attribute name="queue" use="required"/>
+	  <xs:attribute name="queue" use="optional"/>
 	  <xs:attribute name="nodes" use="optional"/>
 	  <xs:attribute name="pes_per_node" use="optional"/>
 	  <xs:attribute name="wallclock" use="required"/>
@@ -84,7 +84,7 @@
     <xs:complexType>
       <xs:simpleContent>
 	<xs:extension base="xs:string">
-	  <xs:attribute name="queue" use="required"/>
+	  <xs:attribute name="queue" use="optional"/>
 	  <xs:attribute name="nodes" use="optional"/>
 	  <xs:attribute name="pes_per_node" use="optional"/>
 	  <xs:attribute name="wallclock" use="required"/>

diff --git a/Makefile b/Makefile
@@ -26,8 +26,8 @@ SUBDIRS = \
 	timeseries \
         conformer \
         conform \
-	diagnostics \
 	ilamb \
+	diagnostics 
 
 # MAKECMDGOALS is the make option: make 'clobber' or 'all'
 TARGET = $(MAKECMDGOALS)

diff --git a/Templates/batch_cheyenne.tmpl b/Templates/batch_cheyenne.tmpl
@@ -1,13 +1,13 @@
+#!/bin/bash
+
 #PBS -N {{ processName }}
 #PBS -q {{ queue }}
 #PBS -l select={{ nodes }}:ncpus={{ ppn }}:mpiprocs={{ ppn }}
 #PBS -l walltime={{ wallclock }}
 #PBS -A {{ project }}
 
-. /glade/u/apps/ch/opt/lmod/7.2.1/lmod/lmod/init/bash
+source /etc/profile.d/modules.sh
 
-export I_MPI_DEVICE=rdma
 export MPI_UNBUFFERED_STDIO=true
 export TMPDIR=$TMPDIR
 
-export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/glade/u/apps/ch/opt/pythonpkgs/2.7/cf_units/1.1.3/gnu/6.2.0/lib
diff --git a/Templates/batch_dav.tmpl b/Templates/batch_dav.tmpl
@@ -0,0 +1,13 @@
+#! /bin/bash -l
+
+#SBATCH -n {{ pes }}
+#SBATCH -N {{ nodes }}
+#SBATCH --ntasks-per-node={{ ppn }}
+#SBATCH -t {{ wallclock }}
+#SBATCH -p dav
+#SBATCH -J {{ processName }}
+#SBATCH -A {{ project }}
+#SBATCH --mem {{ memory }}
+#SBATCH -e {{ processName }}.err.%J
+#SBATCH -o {{ processName }}.out.%J
+#SBATCH -m block
diff --git a/Templates/batch_geyser.tmpl b/Templates/batch_geyser.tmpl
@@ -1,15 +1,13 @@
+#! /bin/bash -l
+
 #SBATCH -n {{ pes }}
 #SBATCH -N {{ nodes }}
 #SBATCH --ntasks-per-node={{ ppn }}
 #SBATCH -t {{ wallclock }}
 #SBATCH -p dav
 #SBATCH -J {{ processName }}
 #SBATCH -A {{ project }}
-#SBATCH -C {{ queue }}
 #SBATCH --mem {{ memory }}
 #SBATCH -e {{ processName }}.err.%J
 #SBATCH -o {{ processName }}.out.%J
-
-source /glade/u/apps/opt/slurm_init/init.sh
-
-export LD_LIBRARY_PATH=/glade/apps/opt/netcdf/4.3.0/gnu/default/lib:$LD_LIBRARY_PATH
+#SBATCH -m block
diff --git a/Templates/cylc_batch_dav.tmpl b/Templates/cylc_batch_dav.tmpl
@@ -0,0 +1,12 @@
+-n {{ pes }}
+-N {{ nodes }}
+--ntasks-per-node={{ ppn }}
+-t {{ wallclock }}
+-p dav
+-J {{ processName }}
+-A {{ project }}
+-C {{ queue }}
+-e {{ processName }}.err.%J
+-o {{ processName }}.out.%J
+--mem {{ memory }}
+-m block
diff --git a/Templates/cylc_batch_geyser.tmpl b/Templates/cylc_batch_geyser.tmpl
@@ -9,3 +9,4 @@
 -e {{ processName }}.err.%J
 -o {{ processName }}.out.%J
 --mem {{ memory }}
+-m block