Merge pull request #115 from bertinia/master

update to pyAverager, timeseries chunking and ocean hi-res NCL

Tools/links.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python
+
+import os
+
+for year in ['0001','0002','0003','0004','0005','0006','0007','0008','0009','0010','0011','0012','0013','0014','0015']:
+    for month in ['01','02','03','04','05','06','07','08','09','10','11','12']:
+
+        srcdir = '/glade/p/cesm0005/archive/b.e20.B1850.f09_g17.pi_control.all.250/ocn/hist'
+        linkdir = '/glade/scratch/aliceb/b.e20.B1850.f09_g17.pi_control.all.250/ocn/hist'
+
+        if not os.path.exists(linkdir):
+            os.makedirs(linkdir)
+
+        casename = 'b.e20.B1850.f09_g17.pi_control.all.250.pop.h'
+        filename = '{0}.{1}-{2}.nc'.format(casename,year,month)
+        os.symlink(os.path.join(srcdir, filename), os.path.join(linkdir, filename))
+
+
+for year in ['0001','0002','0003','0004','0005','0006','0007','0008','0009','0010','0011','0012','0013','0014','0015']:
+    for month in ['01','02','03','04','05','06','07','08','09','10','11','12']:
+
+        srcdir = '/glade/p/cesm0005/archive/b.e20.B1850.f09_g17.pi_control.all.250/ice/hist'
+        linkdir = '/glade/scratch/aliceb/b.e20.B1850.f09_g17.pi_control.all.250/ice/hist'
+
+        if not os.path.exists(linkdir):
+            os.makedirs(linkdir)
+
+        casename = 'b.e20.B1850.f09_g17.pi_control.all.250.cice.h'
+        filename = '{0}.{1}-{2}.nc'.format(casename,year,month)
+        os.symlink(os.path.join(srcdir, filename), os.path.join(linkdir, filename))
+
+
+for year in ['0001','0002','0003','0004','0005','0006','0007','0008','0009','0010','0011','0012','0013','0014','0015']:
+    for month in ['01','02','03','04','05','06','07','08','09','10','11','12']:
+
+        srcdir = '/glade/p/cesm0005/archive/b.e20.B1850.f09_g17.pi_control.all.250/atm/hist'
+        linkdir = '/glade/scratch/aliceb/b.e20.B1850.f09_g17.pi_control.all.250/atm/hist'
+
+        if not os.path.exists(linkdir):
+            os.makedirs(linkdir)
+
+        casename = 'b.e20.B1850.f09_g17.pi_control.all.250.cam.h0'
+        filename = '{0}.{1}-{2}.nc'.format(casename,year,month)
+        os.symlink(os.path.join(srcdir, filename), os.path.join(linkdir, filename))
+
+
+for year in ['0001','0002','0003','0004','0005','0006','0007','0008','0009','0010','0011','0012','0013','0014','0015']:
+    for month in ['01','02','03','04','05','06','07','08','09','10','11','12']:
+
+        srcdir = '/glade/p/cesm0005/archive/b.e20.B1850.f09_g17.pi_control.all.250/lnd/hist'
+        linkdir = '/glade/scratch/aliceb/b.e20.B1850.f09_g17.pi_control.all.250/lnd/hist'
+
+        if not os.path.exists(linkdir):
+            os.makedirs(linkdir)
+
+        casename = 'b.e20.B1850.f09_g17.pi_control.all.250.clm2.h0'
+        filename = '{0}.{1}-{2}.nc'.format(casename,year,month)
+        os.symlink(os.path.join(srcdir, filename), os.path.join(linkdir, filename))
+
+
+for year in ['0001','0002','0003','0004','0005','0006','0007','0008','0009','0010','0011','0012','0013','0014','0015']:
+    for month in ['01','02','03','04','05','06','07','08','09','10','11','12']:
+
+        srcdir = '/glade/p/cesm0005/archive/b.e20.B1850.f09_g17.pi_control.all.250/rof/hist'
+        linkdir = '/glade/scratch/aliceb/b.e20.B1850.f09_g17.pi_control.all.250/rof/hist'
+
+        if not os.path.exists(linkdir):
+            os.makedirs(linkdir)
+
+        casename = 'b.e20.B1850.f09_g17.pi_control.all.250.mosart.h0'
+        filename = '{0}.{1}-{2}.nc'.format(casename,year,month)
+        os.symlink(os.path.join(srcdir, filename), os.path.join(linkdir, filename))
+

averager/pyAverager/README

@@ -1,4 +1,4 @@
-October 26, 2017
+December 13, 2017
 
 ======================================
 
@@ -9,7 +9,7 @@ PyAverager
 A package used for computing averages from climate model output.
 
 Authors:  Sheri Mickelson, Kevin Paul, and John Dennis
-Version:  0.9.12
+Version:  0.9.14
 Copyright:  Contained within LICENSE.txt
 Comments and feedback:  [email protected]
 

averager/pyAverager/pyaverager/PyAverager.py

@@ -388,7 +388,7 @@ def compute_averages(self,spec):
 					# The mean diff rsm function will send the variables once they are created 
 					var_avg_results,var_DIFF_results,var_RMS_results = climAverager.mean_diff_rms(var,region_name,region_num,spec.region_nc_var,
 					    spec.region_wgt_var,years,hist_dict,ave_t.average_types[ave_descr[0]],file_dict,obs_file,
-					    reg_obs_file,inter_comm,spec.serial,VNAME_TAG,AVE_TAG,nlev)
+					    reg_obs_file,inter_comm,spec.serial,VNAME_TAG,AVE_TAG,spec.vertical_levels)
 				    else:
 					if ('__metaChar' in orig_var):
 					    # Handle special meta

averager/pyAverager/pyaverager/__init__.py

@@ -2,5 +2,5 @@
 
 import PyAverager, specification, PreProc
 
-__version__ = "0.9.12"
+__version__ = "0.9.14"
 

averager/pyAverager/pyaverager/climAverager.py

@@ -120,7 +120,10 @@ def avg_var_missing(var,years,hist_dict,ave_info,file_dict,ave_type,fillValue,ti
                 pull_year = yr
             var_val = rover.fetch_slice(hist_dict,pull_year,m,var,file_dict)
             timer.stop("Variable fetch time")
-            var_filled = var_val.filled(fill_value=0) # zero out the masked grid points
+            if (hasattr(var_val, 'filled')):
+                var_filled = var_val.filled(fill_value=0) # zero out the masked grid points
+            else:
+                var_filled = np.ones(var_val.shape)
             # Get and add mask values to the mask accumulator
             if (first_mask):
                 if (MA.any(MA.getmask(var_val))):
@@ -282,8 +285,12 @@ def weighted_avg_var_missing(var,years,hist_dict,ave_info,file_dict,ave_type,fil
             else:
                 pull_year = yr
             var_val = rover.fetch_slice(hist_dict,pull_year,m,var,file_dict)
-            timer.stop("Variable fetch time") 
-            var_filled = var_val.filled(fill_value=0) # zero out the masked grid points
+            timer.stop("Variable fetch time")
+
+            if (hasattr(var_val, 'filled')):
+                var_filled = var_val.filled(fill_value=0) # zero out the masked grid points
+            else:
+                var_filled = np.ones(var_val.shape)
             # Get and add mask values to the mask accumulator
             if (first_mask):
                 if (MA.any(MA.getmask(var_val))):

averager/pyAverager/pyaverager/climFileIO.py

@@ -443,7 +443,8 @@ def define_ave_file(l_master,serial,var_list,lvar_list,meta_list,hist_dict,hist_
         # Add meta variables
         temp = {}
         for mv in meta_list:
-            temp[mv] = create_meta_var(my_file[first_fn],mv,new_file,collapse_dim)
+            if mv in my_file[first_fn].variables.keys():
+                temp[mv] = create_meta_var(my_file[first_fn],mv,new_file,collapse_dim)
         all_files_vars = temp
 
     # Have each rank open it's own variable file(s), retreive variable info, and send to root to create variable 
@@ -515,7 +516,8 @@ def define_ave_file(l_master,serial,var_list,lvar_list,meta_list,hist_dict,hist_
         all_files_vars.update(new_file.variables)
         # All vars are defined, Write all meta vars to the files
         for mv in meta_list:
-            write_meta(all_files_vars, mv, my_file[first_fn])
+            if mv in my_file[first_fn].variables.keys():
+                write_meta(all_files_vars, mv, my_file[first_fn])
     return all_files_vars,new_file
 
 #==============================================================================

averager/pyAverager/setup.py

@@ -3,7 +3,7 @@
 from distutils.core import setup
 
 setup(name='PyAverager',
-      version='0.9.12',
+      version='0.9.14',
       description='Parallel Python Averager for Climate Data',
       author='Sheri Mickelson',
       author_email='[email protected]',

ocn_diag/ncl_lib/compute_rho_cntl.ncl

@@ -31,7 +31,7 @@ begin
 
   do n = 0, nt - 1
     do k = 0, nz - 1 
-      print("month = " + (n+1) + " level = " + (k+1))
+;;      print("month = " + (n+1) + " level = " + (k+1))
       opt = False
       PD(n,k,:,:) = (eos(TEMP(n,k,:,:),SALT(n,k,:,:),0,opt) - 1.0d) * 1000.0d
     end do

ocn_diag/ncl_lib/field_2d.ncl

@@ -4,11 +4,26 @@ load "$NCLPATH/get_environment.ncl"
 load "$NCLPATH/contour_plot.ncl"
 
 begin
-  field_name  = [/"HMXL", "HBLT", "SU", "SV", "BSF" /]
-
+;;  field_name  = [/"HMXL", "HBLT", "SU", "SV", "BSF" /]
+  field_name = NewList("lifo")
   missing = 1.0e30
 
   fileid = addfile(file_netcdf,"r")
+  if (isfilevar(fileid,"HMXL")) then
+    ListPush(field_name,"HMXL")
+  end if
+  if (isfilevar(fileid,"HBLT")) then
+    ListPush(field_name,"HBLT")
+  end if
+  if (isfilevar(fileid,"SU")) then
+    ListPush(field_name,"SU")
+  end if
+  if (isfilevar(fileid,"SV")) then
+    ListPush(field_name,"SV")
+  end if
+  if (isfilevar(fileid,"BSF")) then
+    ListPush(field_name,"BSF")
+  end if
   if (isfilevar(fileid,"DIA_DEPTH")) then
     ListPush(field_name,"DIA_DEPTH")
   end if