Merge pull request #617 from LBL-EESA/develop

4.1.0 release, merge develop in to master

.gitignore

@@ -2,3 +2,6 @@
 *.patch
 _build
 *.pt
+*.vscode*
+.DS_Store
+generated_rtd*

.readthedocs.yml → .readthedocs.yaml

@@ -4,10 +4,13 @@
 
 version: 2
 
+build:
+  image: latest
+
 sphinx:
   configuration: doc/rtd/conf.py
 
 python:
-  version: 3.7
+  version: 3.8
   install:
     - requirements: doc/rtd/requirements.txt

.travis.yml

@@ -17,7 +17,7 @@ env:
     - BUILD_TYPE=Debug
     - TECA_DIR=/travis_teca_dir
     - TECA_PYTHON_VERSION=3
-    - TECA_DATA_REVISION=101
+    - TECA_DATA_REVISION=117
   jobs:
     - DOCKER_IMAGE=ubuntu IMAGE_VERSION=20.04 IMAGE_NAME=ubuntu_20_04 REQUIRE_NETCDF_MPI=TRUE
     - DOCKER_IMAGE=ubuntu IMAGE_VERSION=20.04 IMAGE_NAME=ubuntu_20_04 REQUIRE_NETCDF_MPI=FALSE

CMake/teca_app.cmake

@@ -31,6 +31,7 @@ function (teca_add_app app_name)
                 teca_system teca_core teca_data teca_io teca_alg
                 ${APP_LIBS})
         endif()
+        set_target_properties(${app_name} PROPERTIES APP_TYPE C++)
         install(TARGETS ${app_name} RUNTIME DESTINATION ${BIN_PREFIX})
     else()
         message(STATUS "command line application ${app_name} -- disabled")

CMake/teca_python.cmake

@@ -60,6 +60,8 @@ function (teca_add_python_app app_name)
             if (NOT APP_SOURCES)
                 set(APP_SOURCES "${app_name}.in")
             endif()
+            add_custom_target(${app_name})
+            set_target_properties(${app_name} PROPERTIES APP_TYPE Python)
             teca_py_install_apps(${APP_SOURCES})
         else()
             message(STATUS "command line application ${app_name} -- disabled")

CMake/teca_test.cmake

@@ -51,3 +51,9 @@ function (teca_add_test T_NAME)
         endif()
     endif()
 endfunction()
+
+function (teca_add_app_test T_NAME T_TARGET)
+    if (TARGET ${T_TARGET})
+        teca_add_test(${T_NAME} ${ARGV})
+    endif()
+endfunction()

CMakeLists.txt

@@ -387,26 +387,50 @@ if (BUILD_TESTING)
     # figure out how many cores we can use for parallel tests
     set(TECA_TEST_CORES 0 CACHE STRING
         "Max number of cores for use in parallel tests")
+
+    # by default assume 2 hyperthreads per core, if this is not
+    # the case override here
+    set(HYPERTHREADS_PER_CORE 2 CACHE STRING
+        "The number of hyperthreads per core.")
+
+    # use CMake to get the number of logical cores. includes hyperthreads
+    # in the count.
     if (TECA_TEST_CORES LESS 1)
         ProcessorCount(LOGICAL_CORES)
         if (LOGICAL_CORES EQUAL 0)
-            set(LOGICAL_CORES 4)
+            message(FATAL_ERROR "Failed to detect the number of cores. "
+                "Set TECA_TEST_CORES")
         endif()
     else()
-        math(EXPR LOGICAL_CORES "${TECA_TEST_CORES}*2")
+        math(EXPR LOGICAL_CORES "${TECA_TEST_CORES}*${HYPERTHREADS_PER_CORE}")
     endif()
-    math(EXPR PHYSICAL_CORES "${LOGICAL_CORES}/2")
-    if (PHYSICAL_CORES LESS 3)
-        set(TEST_CORES 2)
-        set(HALF_TEST_CORES 2)
-        set(TWICE_TEST_CORES 4)
-    else()
-        set(TEST_CORES ${PHYSICAL_CORES})
-        math(EXPR HALF_TEST_CORES "${TEST_CORES}/2")
-        set(TWICE_TEST_CORES ${LOGICAL_CORES})
+
+    # adjust count for hyperthreads.
+    math(EXPR PHYSICAL_CORES "${LOGICAL_CORES}/${HYPERTHREADS_PER_CORE}")
+    if (PHYSICAL_CORES LESS 1)
+        message(FATAL_ERROR "Invalid CPU configuration. "
+            "LOGICAL_CORES=${LOGICAL_CORES} HYPERTHREADS_PER_CORE="
+            "${HYPERTHREADS_PER_CORE}")
     endif()
+
+    # set the number of cores to use for pure MPI or purely threaded tests
+    set(TEST_CORES ${PHYSICAL_CORES})
     message(STATUS "regression testing -- enabled (${TEST_CORES} cores).")
 
+    # set the number of cores to use for MPI + threads tests. if there are too
+    # few physical cores then disable hybrid parallel tests
+    math(EXPR HALF_TEST_CORES "${TEST_CORES}/2")
+    if (HALF_TEST_CORES LESS 2)
+        message(STATUS "Hybrid parallel tests -- disabled.")
+        set(TEST_MPI_THREADS OFF)
+    else()
+        message(STATUS "Hybrid parallel tests -- enabled.")
+        set(TEST_MPI_THREADS ON)
+    endif()
+
+    # set the number of cores for oversubscription/streaming tests
+    math(EXPR TWICE_TEST_CORES "${TEST_CORES}*2")
+
     add_subdirectory(test)
 else()
     message(STATUS "regression testing -- disbaled")

README.md

@@ -1,4 +1,8 @@
-<img src="doc/rtd/images/teca_logo_crop2_lg.png" width="48%">
+<table style="border: 1px; border-collapse: collapse; border-spacing: 0px;">
+<tr><td width="402px">
+<img src="https://raw.githubusercontent.com/LBL-EESA/TECA/8ea6a121c8c29cdab31be4226b0564c9ee5a9726/doc/rtd/images/teca_logo_crop2_lg.png" width="400px">
+</td></tr>
+</table>
 <a href="https://travis-ci.com/LBL-EESA/TECA"><img src="https://travis-ci.com/LBL-EESA/TECA.svg?token=zV3LhFtYvjcvo67W2uji&branch=master"></a>
 <a href="https://teca.readthedocs.io/en/latest/?badge=latest"><img src="https://readthedocs.org/projects/teca/badge/?version=latest"></a>
 <a href="https://badge.fury.io/py/teca"><img src="https://badge.fury.io/py/teca.svg" alt="PyPI version"></a>
@@ -8,7 +12,7 @@
 TECA is a collection of climate analysis algorithms geared toward extreme event detection and tracking implemented in a scalable parallel framework. The code has been successfully deployed and run at massive scales on current DOE supercomputers. TECA's core is written in modern C++ and exploits MPI + X parallelism where X is one of threads, OpenMP, or GPUs. The framework supports a number of parallel design patterns including distributed data parallelism and map-reduce. While modern C++ delivers the highest performance, Python bindings make the code approachable and easy to use.
 
 ### Documentation
-The [TECA User's Guide](https://teca.readthedocs.io/en/latest/) is the authorotative source for documentation on topics such as [installing TECA](https://teca.readthedocs.io/en/latest/installation.html), running TECA's [command line applications](https://teca.readthedocs.io/en/latest/applications.html), and [Python development](https://teca.readthedocs.io/en/latest/python.html).
+The [TECA User's Guide](https://teca.readthedocs.io/en/latest/) is the authorotative source for documentation on topics such as [installing TECA](https://teca.readthedocs.io/en/latest/installation.html), running TECA's [command line applications](https://teca.readthedocs.io/en/latest/applications.html), and [Python development](https://teca.readthedocs.io/en/latest/python.html). The TECA source code is documented on our [Doxygen site](https://teca.readthedocs.io/en/latest/doxygen/index.html).
 
 ### Tutorials
 The [TECA tutorials](https://sourceforge.net/p/teca/TECA_tutorials) subversion repository contains slides from previous tutorials.

alg/CMakeLists.txt

@@ -21,16 +21,20 @@ set(teca_alg_cxx_srcs
     teca_component_statistics.cxx
     teca_derived_quantity.cxx
     teca_descriptive_statistics.cxx
+    teca_elevation_mask.cxx
     teca_evaluate_expression.cxx
     teca_face_to_cell_centering.cxx
     teca_geography.cxx
+    teca_indexed_dataset_cache.cxx
     teca_integrated_vapor_transport.cxx
+    teca_integrated_water_vapor.cxx
     teca_l2_norm.cxx
     teca_latitude_damper.cxx
     teca_laplacian.cxx
     teca_mask.cxx
     teca_normalize_coordinates.cxx
     teca_parser.cxx
+    teca_rename_variables.cxx
     teca_table_calendar.cxx
     teca_table_reduce.cxx
     teca_table_region_mask.cxx
@@ -41,7 +45,8 @@ set(teca_alg_cxx_srcs
     teca_tc_classify.cxx
     teca_tc_wind_radii.cxx
     teca_tc_trajectory.cxx
-    teca_temporal_average.cxx
+    teca_simple_moving_average.cxx
+    teca_unpack_data.cxx
     teca_valid_value_mask.cxx
     teca_variant_array_operand.cxx
     teca_vertical_coordinate_transform.cxx

alg/teca_2d_component_area.cxx

@@ -132,18 +132,22 @@ void teca_2d_component_area::get_properties_description(
             "name of the varibale containing region labels")
         TECA_POPTS_GET(int, prefix, contiguous_component_ids,
             "when the region label ids start at 0 and are consecutive "
-            "this flag enables use of an optimization (0)")
+            "this flag enables use of an optimization")
         TECA_POPTS_GET(long, prefix, background_id,
-            "the label id that corresponds to the background (-1)")
+            "the label id that corresponds to the background")
         ;
 
+    this->teca_algorithm::get_properties_description(prefix, opts);
+
     global_opts.add(opts);
 }
 
 // --------------------------------------------------------------------------
 void teca_2d_component_area::set_properties(const std::string &prefix,
     variables_map &opts)
 {
+    this->teca_algorithm::set_properties(prefix, opts);
+
     TECA_POPTS_SET(opts, std::string, prefix, component_variable)
     TECA_POPTS_SET(opts, int, prefix, contiguous_component_ids)
     TECA_POPTS_SET(opts, long, prefix, background_id)

alg/teca_2d_component_area.h

@@ -10,40 +10,40 @@
 
 TECA_SHARED_OBJECT_FORWARD_DECL(teca_2d_component_area)
 
-/// an algorithm that computes the area of labeled regions
+/// An algorithm that computes the areas of labeled regions
 /**
-Given a set of labels on a Cartesian mesh, the algorithm computes the area of
-each region. Regions are identified by assigning a unique integer value to each
-mesh point that belongs in the region. The component_variable property names
-the variable containing the region labels.
-
-if the region labels start at 0 and are contiguous then an optimization can be
-used. Set contiguous_component_ids property to enable the optimization. Note that
-TECA's connected component labeler assigns the background (i.e. cells not inside
-the segmentation) the label 0. One can identify the background region and area
-via this label. When processing data generated outside of TECA it might be
-necessary to supply the background label. Use -2 if there is no background.
-
-the input dataset is passed through and the results of the calculations are
-stored in the output dataset metadata in the following keys:
-
-    number_of_components - number of component ids for which area was
-                           computed. Note that this can include a background
-                           component i.e. for cells outside of the segmentation.
-
-    component_ids - a vector containing the label of each component. This is
-                    always starts with 0, where the label 0 identifies cells
-                    out side of the segmentation, and ranges up to
-                    number_of_components - 1, where the labels from 1 up to
-                    number_of_components - 1 identify connected regions of
-                    cells inside the segmentation.
-
-    component_area - a vector containing the area for the corresponding entry
-                     in the component_ids array.
-
-    background_id - the label used for cells outside of the segmentation,
-                    i.e. the background. This can be used to skip processing
-                    of the background when desirable.
+ * Given a set of labels on a Cartesian mesh, the algorithm computes the area
+ * of each region. Regions are identified by assigning a unique integer value
+ * to each mesh point that belongs in the region. The component_variable
+ * property names the variable containing the region labels.
+ *
+ * if the region labels start at 0 and are contiguous then an optimization can
+ * be used. Set contiguous_component_ids property to enable the optimization.
+ * Note that TECA's connected component labeler assigns the background (i.e.
+ * cells not inside the segmentation) the label 0. One can identify the
+ * background region and area via this label. When processing data generated
+ * outside of TECA it might be necessary to supply the background label. Use -2
+ * if there is no background.
+ *
+ * the input dataset is passed through and the results of the calculations are
+ * stored in the output dataset metadata in the following keys:
+ *
+ *  | name                 | description |
+ *  | ----                 | ----------- |
+ *  | number_of_components | number of component ids for which area was |
+ *  |                      | computed. Note that this can include a background |
+ *  |                      | component i.e. for cells outside of the segmentation. |
+ *  | component_ids        | a vector containing the label of each component. This is |
+ *  |                      | always starts with 0, where the label 0 identifies |
+ *  |                      | cells out side of the segmentation, and ranges up |
+ *  |                      | to number_of_components - 1, where the labels from |
+ *  |                      | 1 up to number_of_components - 1 identify |
+ *  |                      | connected regions of cells inside the segmentation. |
+ *  | component_area       | a vector containing the area for the corresponding |
+ *  |                      |  entry in the component_ids array. |
+ *  | background_id        | the label used for cells outside of the segmentation, |
+ *  |                      | i.e. the background. This can be used to skip processing |
+ *  |                      | of the background when desirable. |
 */
 class teca_2d_component_area : public teca_algorithm
 {
@@ -53,23 +53,40 @@ class teca_2d_component_area : public teca_algorithm
     TECA_ALGORITHM_CLASS_NAME(teca_2d_component_area)
     ~teca_2d_component_area();
 
-    // report/initialize to/from Boost program options objects.
+    /** @name program_options
+     * report/initialize to/from Boost program options objects.
+     */
+    ///@{
     TECA_GET_ALGORITHM_PROPERTIES_DESCRIPTION()
     TECA_SET_ALGORITHM_PROPERTIES()
+    ///@}
 
-    // set the name of the input array
+    /** @name component_variable
+     * Sets the name of the array containing component labels to compute the
+     * area of.
+     */
+    ///@{
     TECA_ALGORITHM_PROPERTY(std::string, component_variable)
+    ///@}
 
-    // set this only if you know for certain that label ids are contiguous and
-    // start at 0. this enables use of a faster implementation.
+    /** @name contiguous_component_ids
+     * set this only if you know for certain that label ids are contiguous and
+     * start at 0. this enables use of a faster implementation.
+     */
+    ///@{
     TECA_ALGORITHM_PROPERTY(int, contiguous_component_ids)
-
-    // set this to override the component label used for background. By default
-    // this is set to -1 to indicate that the value should be obtained from the
-    // metadata key `background_id`.  Note that TECA's connected component
-    // labeler uses the id 0 for the background and passes this in a metadata
-    // key and as a result no action is required.
+    ///@}
+
+    /** @name background_id
+     * set this to override the component label used for background. By default
+     * this is set to -1 to indicate that the value should be obtained from the
+     * metadata key `background_id`.  Note that TECA's connected component
+     * labeler uses the id 0 for the background and passes this in a metadata
+     * key and as a result no action is required.
+     */
+    ///@{
     TECA_ALGORITHM_PROPERTY(long, background_id)
+    ///@}
 
 protected:
     teca_2d_component_area();