Merge pull request #597 from LBL-EESA/cf_writer_intervals

cf_writer intervals

.travis.yml

@@ -17,7 +17,7 @@ env:
     - BUILD_TYPE=Debug
     - TECA_DIR=/travis_teca_dir
     - TECA_PYTHON_VERSION=3
-    - TECA_DATA_REVISION=115
+    - TECA_DATA_REVISION=116
   jobs:
     - DOCKER_IMAGE=ubuntu IMAGE_VERSION=20.04 IMAGE_NAME=ubuntu_20_04 REQUIRE_NETCDF_MPI=TRUE
     - DOCKER_IMAGE=ubuntu IMAGE_VERSION=20.04 IMAGE_NAME=ubuntu_20_04 REQUIRE_NETCDF_MPI=FALSE

alg/teca_tc_candidates.cxx

@@ -4,7 +4,6 @@
 #include "teca_variant_array.h"
 #include "teca_table.h"
 #include "teca_database.h"
-#include "teca_calendar.h"
 #include "teca_coordinate_util.h"
 #include "gfdl_tc_candidates.h"
 

alg/teca_temporal_reduction.py

@@ -181,7 +181,7 @@ def __next__(self):
                 sm = self.month
 
                 t0 = '%04d-%02d-01 00:00:00' % (sy, sm)
-                i0 = coordinate_util.time_step_of(self.t, True, True,
+                i0 = coordinate_util.time_step_of(self.t, False, True,
                                                   self.calendar,
                                                   self.units, t0)
 
@@ -259,7 +259,7 @@ def __next__(self):
                 month = self.month
 
                 t0 = '%04d-%02d-01 00:00:00' % (self.year, self.month)
-                i0 = coordinate_util.time_step_of(self.t, True, True,
+                i0 = coordinate_util.time_step_of(self.t, False, True,
                                                   self.calendar,
                                                   self.units, t0)
 
@@ -350,7 +350,7 @@ def __next__(self):
                 t0 = '%04d-%02d-%02d 00:00:00' % \
                     (self.year, self.month, self.day)
 
-                i0 = coordinate_util.time_step_of(self.t, True, True,
+                i0 = coordinate_util.time_step_of(self.t, False, True,
                                                   self.calendar,
                                                   self.units, t0)
 

apps/teca_bayesian_ar_detect.cpp

@@ -129,7 +129,11 @@ int main(int argc, char **argv)
             "\nA path and file name pattern for the output NetCDF files. %t% is replaced with a"
             " human readable date and time corresponding to the time of the first time step in"
             " the file. Use --cf_writer::date_format to change the formatting\n")
-
+        ("file_layout", value<std::string>()->default_value("monthly"),
+            "\nSelects the size and layout of the set of output files. May be one of"
+            " number_of_steps, daily, monthly, seasonal, or yearly. Files are structured"
+            " such that each file contains one of the selected interval. For the number_of_steps"
+            " option use --steps_per_file.\n")
         ("steps_per_file", value<long>()->default_value(128),
             "\nnumber of time steps per output file\n")
 
@@ -251,6 +255,7 @@ int main(int argc, char **argv)
     cf_writer->set_verbose(0);
     cf_writer->set_thread_pool_size(1);
     cf_writer->set_steps_per_file(128);
+    cf_writer->set_layout(teca_cf_writer::monthly);
 
     // package basic and advanced options for display
     options_description all_opt_defs(help_width, help_width - 4);
@@ -564,6 +569,14 @@ int main(int argc, char **argv)
     if (!opt_vals["last_step"].defaulted())
         cf_writer->set_last_step(opt_vals["last_step"].as<long>());
 
+    if (!opt_vals["file_layout"].defaulted() &&
+        cf_writer->set_layout(opt_vals["file_layout"].as<std::string>()))
+    {
+        TECA_ERROR("An invalid file layout was provided \""
+            << opt_vals["file_layout"].as<std::string>() << "\"")
+        return -1;
+    }
+
     if (opt_vals.count("verbose"))
     {
         ar_detect->set_verbose(1);

apps/teca_cf_restripe.cpp

@@ -68,6 +68,12 @@ int main(int argc, char **argv)
             " human readable date and time corresponding to the time of the first time step in"
             " the file. Use --cf_writer::date_format to change the formatting\n")
 
+        ("file_layout", value<std::string>()->default_value("monthly"),
+            "\nSelects the size and layout of the set of output files. May be one of"
+            " number_of_steps, daily, monthly, seasonal, or yearly. Files are structured"
+            " such that each file contains one of the selected interval. For the number_of_steps"
+            " option use --steps_per_file.\n")
+
         ("steps_per_file", value<long>(), "\nnumber of time steps per output file\n")
 
         ("normalize_coordinates", "\nEnable coordinate normalization pipeline stage\n")
@@ -129,6 +135,7 @@ int main(int argc, char **argv)
 
     p_teca_cf_writer cf_writer = teca_cf_writer::New();
     cf_writer->get_properties_description("cf_writer", advanced_opt_defs);
+    cf_writer->set_layout(teca_cf_writer::monthly);
 
     // Add an executive for the writer
     p_teca_index_executive exec = teca_index_executive::New();
@@ -205,6 +212,14 @@ int main(int argc, char **argv)
         cf_writer->set_information_arrays(
             opt_vals["information_arrays"].as<std::vector<std::string>>());
 
+    if (!opt_vals["file_layout"].defaulted() &&
+        cf_writer->set_layout(opt_vals["file_layout"].as<std::string>()))
+    {
+        TECA_ERROR("An invalid file layout was provided \""
+            << opt_vals["file_layout"].as<std::string>() << "\"")
+        return -1;
+    }
+
     if (opt_vals.count("steps_per_file"))
         cf_writer->set_steps_per_file(
             opt_vals["steps_per_file"].as<long>());

apps/teca_deeplab_ar_detect.in

@@ -87,6 +87,14 @@ parser.add_argument('--output_file', type=str, required=True,
          ' time of the first time step in the file. Use --date_format to change'
          ' the formatting')
 
+parser.add_argument('--file_layout', type=str, default='monthly',
+                    help='Selects the size and layout of the set of output'
+                         ' files. May be one of number_of_steps, daily,'
+                         ' monthly, seasonal, or yearly. Files are structured'
+                         ' such that each file contains one of the selected'
+                         ' interval. For the number_of_steps option use'
+                         ' --steps_per_file.')
+
 parser.add_argument('--steps_per_file', type=int, required=False, default=128,
     help='number of time steps per output file')
 
@@ -241,6 +249,7 @@ writer.set_input_connection(ar_tag.get_output_port())
 writer.set_executive(exe)
 writer.set_thread_pool_size(1)
 writer.set_file_name(args.output_file)
+writer.set_layout(args.file_layout)
 writer.set_steps_per_file(args.steps_per_file)
 writer.set_first_step(args.first_step)
 writer.set_last_step(args.last_step)

apps/teca_integrated_vapor_transport.cpp

@@ -64,13 +64,11 @@ int main(int argc, char **argv)
 
         ("wind_u", value<std::string>()->default_value("U"),
             "\nname of variable with the 3D longitudinal component of the wind vector.\n")
-
         ("wind_v", value<std::string>()->default_value("V"),
             "\nname of variable with the 3D latitudinal component of the wind vector.\n")
 
         ("ivt_u", value<std::string>()->default_value("IVT_U"),
             "\nname to use for the longitudinal component of the integrated vapor transport vector.\n")
-
         ("ivt_v", value<std::string>()->default_value("IVT_V"),
             "\nname to use for the latitudinal component of the integrated vapor transport vector.\n")
 
@@ -91,7 +89,11 @@ int main(int argc, char **argv)
             "\nA path and file name pattern for the output NetCDF files. %t% is replaced with a"
             " human readable date and time corresponding to the time of the first time step in"
             " the file. Use --cf_writer::date_format to change the formatting\n")
-
+        ("file_layout", value<std::string>()->default_value("monthly"),
+            "\nSelects the size and layout of the set of output files. May be one of"
+            " number_of_steps, daily, monthly, seasonal, or yearly. Files are structured"
+            " such that each file contains one of the selected interval. For the number_of_steps"
+            " option use --steps_per_file.\n")
         ("steps_per_file", value<long>()->default_value(128),
             "\nnumber of time steps per output file\n")
 
@@ -211,6 +213,7 @@ int main(int argc, char **argv)
     cf_writer->get_properties_description("cf_writer", advanced_opt_defs);
     cf_writer->set_verbose(0);
     cf_writer->set_steps_per_file(128);
+    cf_writer->set_layout(teca_cf_writer::monthly);
 
     // package basic and advanced options for display
     options_description all_opt_defs(help_width, help_width - 4);
@@ -442,6 +445,14 @@ int main(int argc, char **argv)
     if (!opt_vals["last_step"].defaulted())
         cf_writer->set_last_step(opt_vals["last_step"].as<long>());
 
+    if (!opt_vals["file_layout"].defaulted() &&
+        cf_writer->set_layout(opt_vals["file_layout"].as<std::string>()))
+    {
+        TECA_ERROR("An invalid file layout was provided \""
+            << opt_vals["file_layout"].as<std::string>() << "\"")
+        return -1;
+    }
+
     if (opt_vals.count("verbose"))
     {
         cf_writer->set_verbose(1);

apps/teca_integrated_water_vapor.cpp

@@ -69,7 +69,11 @@ int main(int argc, char **argv)
             "\nA path and file name pattern for the output NetCDF files. %t% is replaced with a"
             " human readable date and time corresponding to the time of the first time step in"
             " the file. Use --cf_writer::date_format to change the formatting\n")
-
+        ("file_layout", value<std::string>()->default_value("monthly"),
+            "\nSelects the size and layout of the set of output files. May be one of"
+            " number_of_steps, daily, monthly, seasonal, or yearly. Files are structured"
+            " such that each file contains one of the selected interval. For the number_of_steps"
+            " option use --steps_per_file.\n")
         ("steps_per_file", value<long>()->default_value(128),
             "\nnumber of time steps per output file\n")
 
@@ -180,6 +184,7 @@ int main(int argc, char **argv)
     cf_writer->get_properties_description("cf_writer", advanced_opt_defs);
     cf_writer->set_verbose(0);
     cf_writer->set_steps_per_file(128);
+    cf_writer->set_layout(teca_cf_writer::monthly);
 
     // package basic and advanced options for display
     options_description all_opt_defs(help_width, help_width - 4);
@@ -344,6 +349,14 @@ int main(int argc, char **argv)
     if (!opt_vals["last_step"].defaulted())
         cf_writer->set_last_step(opt_vals["last_step"].as<long>());
 
+    if (!opt_vals["file_layout"].defaulted() &&
+        cf_writer->set_layout(opt_vals["file_layout"].as<std::string>()))
+    {
+        TECA_ERROR("An invalid file layout was provided \""
+            << opt_vals["file_layout"].as<std::string>() << "\"")
+        return -1;
+    }
+
     if (opt_vals.count("verbose"))
     {
         cf_writer->set_verbose(1);

apps/teca_temporal_reduction.in

@@ -66,6 +66,14 @@ parser.add_argument('--output_file', type=str, required=True,
                          ' step in the file. Use --date_format to change'
                          ' the formatting')
 
+parser.add_argument('--file_layout', type=str, default='yearly',
+                    help='Selects the size and layout of the set of output'
+                         ' files. May be one of number_of_steps, daily,'
+                         ' monthly, seasonal, or yearly. Files are structured'
+                         ' such that each file contains one of the selected'
+                         ' interval. For the number_of_steps option use'
+                         ' --steps_per_file.')
+
 parser.add_argument('--steps_per_file', type=int, default=128,
                     help='number of time steps to write to each output '
                          'file')
@@ -85,7 +93,7 @@ parser.add_argument('--t_axis_variable', type=str, default='time',
                     help='name of the variable to use for t-coordinates')
 
 parser.add_argument('--n_threads', type=int, default=2,
-                    help='Number of threads to use when stremaing the '
+                    help='Number of threads to use when streaming the '
                          'reduction')
 
 parser.add_argument('--verbose', type=int, default=0,
@@ -99,6 +107,7 @@ except Exception:
 
 in_files = args.input_regex
 out_files = args.output_file
+layout = args.file_layout
 steps_per_file = args.steps_per_file
 n_threads = args.n_threads
 interval = args.interval
@@ -115,6 +124,7 @@ verbose = args.verbose
 if verbose and rank == 0:
     sys.stderr.write('running on %d ranks' % (n_ranks))
     sys.stderr.write('n_threads=%d\n' % (n_threads))
+    sys.stderr.write('file_layout=%s\n'%(layout))
     sys.stderr.write('steps_per_file=%d\n' % (steps_per_file))
     sys.stderr.write('interval=%s\n' % (interval))
     sys.stderr.write('operator=%s\n' % (operator))
@@ -152,6 +162,7 @@ cfw = teca_cf_writer.New()
 cfw.set_input_connection(mav.get_output_port())
 cfw.set_verbose(verbose)
 cfw.set_thread_pool_size(1)
+cfw.set_layout(layout)
 cfw.set_steps_per_file(steps_per_file)
 cfw.set_file_name(out_files)
 cfw.set_point_arrays(point_arrays)

core/CMakeLists.txt

@@ -7,7 +7,6 @@ set(teca_core_srcs
     teca_algorithm_executive.cxx
     teca_bad_cast.cxx
     teca_binary_stream.cxx
-    teca_calendar.cxx
     teca_common.cxx
     teca_dataset.cxx
     teca_dataset_source.cxx

data/CMakeLists.txt

@@ -9,6 +9,7 @@ set(teca_data_srcs
     teca_arakawa_c_grid.cxx
     teca_array_attributes.cxx
     teca_array_collection.cxx
+    teca_calendar_util.cxx
     teca_cartesian_mesh.cxx
     teca_coordinate_util.cxx
     teca_curvilinear_mesh.cxx