Merge pull request #191 from bertinia/master

add call to manage_externals/checkout_externals to the create_python_env bash script

averager/pp_tests/atm_averages.o3770948

@@ -0,0 +1,2 @@
+Running from virtualenv directory:
+/gpfs/fs1/work/aliceb/sandboxes/dev/postprocessing_new/cesm-env2/bin

averager/pp_tests/runAvg_mpi_atm.sh

@@ -0,0 +1,68 @@
+#!/bin/bash
+
+#PBS -N atm_averages
+#PBS -q regular
+#PBS -l select=2:ncpus=9:mpiprocs=9
+#PBS -l walltime=00:05:00
+#PBS -A P93300606
+
+source /etc/profile.d/modules.sh
+
+export MPI_UNBUFFERED_STDIO=true
+export TMPDIR=$TMPDIR
+
+
+##########
+##
+## See https://github.com/NCAR/CESM_postprocessing/wiki for details
+## regarding settings for optimal performance for CESM postprocessing tools.
+##
+##########
+
+if [ ! -e /gpfs/fs1/work/aliceb/sandboxes/dev/postprocessing_new/cesm-env2/bin ]; then
+    echo "*************************************************************************************"
+    echo "CESM atm_averages exiting due to non-existant python virtual environment in"
+    echo "    /gpfs/fs1/work/aliceb/sandboxes/dev/postprocessing_new/cesm-env2/bin"
+    echo "You must first run:"
+    echo "$POSTPROCESS_PATH/create_python_env -machine [machine]"
+    echo "*************************************************************************************"
+    exit
+fi
+
+
+module purge
+
+
+
+
+## activate the virtualenv that contains all the non-bootstrapped dependencies
+
+cd /gpfs/fs1/work/aliceb/sandboxes/dev/postprocessing_new/cesm-env2/bin
+echo "Running from virtualenv directory:"
+pwd
+. activate
+
+## load the boot-strap modules 
+
+
+module load python/2.7.14
+
+module load intel/17.0.1
+
+module load ncarenv
+
+module load ncarcompilers
+
+module load mpt/2.15f
+
+module load netcdf/4.6.1
+
+module load nco/4.7.4
+
+module load ncl/6.4.0
+
+export POSTPROCESS_PATH=/gpfs/fs1/work/aliceb/sandboxes/dev/postprocessing_new
+
+mpiexec_mpt dplace -s 1 /gpfs/fs1/work/aliceb/sandboxes/dev/postprocessing_new/averager/pp_tests/test_atm_series.py >> /gpfs/fs1/work/aliceb/sandboxes/dev/postprocessing_new/averager/pp_tests/atm_series.log 2>&1
+
+

averager/pp_tests/test_atm_series.py

@@ -0,0 +1,74 @@
+#!/usr/bin/env python2
+
+from __future__ import print_function
+import sys
+
+# check the system python version and require 2.7.x or greater                                                                                                              
+if sys.hexversion < 0x02070000:
+    print(70 * '*')
+    print('ERROR: {0} requires python >= 2.7.x. '.format(sys.argv[0]))
+    print('It appears that you are running python {0}'.format(
+            '.'.join(str(x) for x in sys.version_info[0:3])))
+    print(70 * '*')
+    sys.exit(1)
+
+import os
+
+#
+# check the POSTPROCESS_PATH which must be set
+#
+try:
+    os.environ["POSTPROCESS_PATH"]
+except KeyError:
+    err_msg = ('create_postprocess ERROR: please set the POSTPROCESS_PATH environment variable.' \
+                   ' For example on yellowstone: setenv POSTPROCESS_PATH /glade/p/cesm/postprocessing')
+    raise OSError(err_msg)
+
+cesm_pp_path = os.environ["POSTPROCESS_PATH"]
+
+#
+# activate the virtual environment that was created by create_python_env
+#
+if not os.path.isfile('{0}/cesm-env2/bin/activate_this.py'.format(cesm_pp_path)):
+    err_msg = ('create_postprocess ERROR: the virtual environment cesm-env2 does not exist.' \
+                   ' Please run $POSTPROCESS_PATH/create_python_env -machine [machine name]')
+    raise OSError(err_msg)
+
+execfile('{0}/cesm-env2/bin/activate_this.py'.format(cesm_pp_path), dict(__file__='{0}/cesm-env2/bin/activate_this.py'.format(cesm_pp_path)))
+
+from pyaverager import PyAverager, specification
+
+#### User modify ####
+
+in_dir='/glade/scratch/aliceb/F1850C5_f02_cntrl/atm//proc/tseries/month_1'
+out_dir= '/glade/scratch/aliceb/F1850C5_f02_cntrl/atm/proc/climo/F1850C5_f02_cntrl//F1850C5_f02_cntrl.2-5'
+pref= 'F1850C5_f02_cntrl.cam.h0'
+htype= 'series'
+average = ['dep_ann:2:5', 'dep_djf:2:5', 'dep_mam:2:5', 'dep_jja:2:5', 'dep_son:2:5', 'jan:2:5', 'feb:2:5', 'mar:2:5', 'apr:2:5', 'may:2:5', 'jun:2:5', 'jul:2:5', 'aug:2:5', 'sep:2:5', 'oct:2:5', 'nov:2:5', 'dec:2:5']
+collapse_dim = 'lon'
+wght= False
+ncfrmt = 'netcdfLarge'
+serial=False
+suffix = 'nc'
+clobber = True
+date_pattern= 'yyyymm-yyyymm'
+var_list = ['CLDLIQ', 'PRECC', 'T', 'TS']
+
+#### End user modify ####
+
+pyAveSpecifier = specification.create_specifier(in_directory=in_dir,
+			          out_directory=out_dir,
+				  prefix=pref,
+                                  suffix=suffix,
+                                  date_pattern=date_pattern,
+				  hist_type=htype,
+				  avg_list=average,
+				  varlist=var_list,
+                                  collapse_dim=collapse_dim,
+				  weighted=wght,
+				  ncformat=ncfrmt,
+				  serial=serial,
+                                  clobber=clobber)
+
+PyAverager.run_pyAverager(pyAveSpecifier)
+

create_python_env

@@ -15,7 +15,7 @@ function Usage {
     echo ""
     echo "     This script sets up the python virtual environment (env) for a given supported machine."
     echo "     This script executes the following steps:"
-    echo "     - loads the python modules for this machine necessary to boot-strap the env"
+    echo "     - clones the NCAR Python Library (NPL) for this machine necessary to boot-strap the env"
     echo "     - make env"
     echo "     - activate env"
     echo "     - install post-processing tools into the env"
@@ -152,23 +152,15 @@ and rerun this script."
     exit 0
 fi
 
+#----------------------------------------------------------------------
+# install the external tools via the manage_externals/checkout_externals
+#----------------------------------------------------------------------
+./manage_externals/checkout_externals 
+
 curdir=`pwd`
 echo $curdir
 cd $pp_dir
 
-#----------------------------------------------------------------------
-# create the virtual environment. Makefile checks to see if it is
-# already setup, so only done once per case.
-#----------------------------------------------------------------------
-##echo "$progname - making virtual environment in ${pp_dir}/cesm-env2."
-##if [ ${machine} != geyser ]; then
-##    make env
-##    if [ $? -ne 0 ]; then
-##	echo "ERROR: Unable to create virtual environment in ${pp_dir}/cesm-env2. Exiting..."
-##	exit 1
-##    fi
-##fi
-
 #----------------------------------------------------------------------
 # activate virtualenv for remainder of this script
 #----------------------------------------------------------------------

diagnostics/diagnostics/ocn/Plots/ecosystem_maps.py

@@ -203,7 +203,7 @@ def __init__(self):
         self._columns_s2 = [3]
         self._columns_s3 = [6]
 
-        self._columnns = [ self._columns_s1, self._columns_s2, self._columns_s3 ]
+        self._columns = [ self._columns_s1, self._columns_s2, self._columns_s3 ]
 
         self._plotTitles = ['Ecosystem: Maps', 'Ecosystem: Maps at Depth (with obs where applicable)']
         self._labels = ['NO3','PO4','SiO3','O2','DIC','ALK','Fe']

diagnostics/diagnostics/ocn/ocn_diags_generator.py

@@ -109,7 +109,7 @@ def setup_diags(envDict):
     diag_dict = dict()
     # ocean bgc being computed by IOMB
 ##    avail_diags = ['MODEL_VS_OBS', 'MODEL_VS_OBS_ECOSYS', 'MODEL_VS_CONTROL', 'MODEL_VS_CONTROL_ECOSYS', 'MODEL_TIMESERIES', 'MODEL_TIMESERIES_ECOSYS']
-    avail_diags = ['MODEL_VS_OBS', 'MODEL_VS_CONTROL', 'MODEL_TIMESERIES']
+    avail_diags = ['MODEL_VS_OBS', 'MODEL_VS_OBS_ECOSYS', 'MODEL_VS_CONTROL', 'MODEL_TIMESERIES']
     for diag in avail_diags:
         diag_dict[diag] = False
         for key, value in envDict.iteritems():

ocn_diag/eco_lib/makefile

@@ -1,7 +1,7 @@
 all:	remap
 
 remap:
-	f2py -c --fcompiler=gfortran --f77exec=/usr/bin/gfortran -m remap remap.f
+	f2py -c --fcompiler=intelem -m remap remap.f
 
 clean:
 	rm -r *.so